Support 202309 (#1458)

* fixes #1260 - playwright tests
fixes #1394 - v2 search failing to find some names
fixes #1374 - Search to include new objects: Campus & Carrier
fixes #1164 - better rdap error reporting
fixes #1368 - Facility data export into Google Earth KMZ
fixes #1328 - Support web updates from a source of truth
fixes #1257 - Help text covers non-compliant email addresses
fixes #1313 - Improve email confirmation control - add 3 month option & maybe set new default value
fixes #1380 - Reset 'Social Media' to '[]' if field has no value

* linting

* remove target=_blank

* bump ES version to 8.10

* Cache and ES updates (#1459)

* elasticsearch major version pin and relock

* set decimal fields to python value on client save for load_data

* force use of redis password

* add default_meta to render

* add generated, clean up var names

* run pre-commit

* update ES for https and password

* rm cruft

* isort

---------

Co-authored-by: 20C <code@20c.com>
Co-authored-by: Matt Griswold <grizz@20c.com>

ux-tests/config.beta.example.json

@@ -0,0 +1,18 @@
+{
+    "url": "https://beta.peeringdb.com",
+    "accounts":{
+    },
+    "test_search_exchange": {
+        "name": "ChIX"
+    },
+    "test_search_network": {
+        "name": "20C",
+        "quick_search_result": "20C (63311)"
+    },
+    "test_search_facility": {
+        "name": "CoreSite - Chicago (CH1)"
+    },
+    "test_search_organization": {
+        "name": "20C, LLC"
+    }
+}

ux-tests/config.dev.example.json

@@ -0,0 +1,31 @@
+{
+    "url": "https://localhost:8000",
+    "accounts":{
+        "user": {
+            "username": "<username>",
+            "password": "<password>"
+        }
+    },
+    "test_search_exchange": {
+        "name": "ChIX"
+    },
+    "test_search_network": {
+        "name": "20C",
+        "quick_search_result": "20C (63311)"
+    },
+    "test_search_facility": {
+        "name": "CoreSite - Chicago (CH1)"
+    },
+    "test_search_organization": {
+        "name": "20C, LLC"
+    },
+    "test_add_api_key": {
+        "description": "test"
+    },
+    "test_delete_api_key": {
+        "description": "test"
+    },
+    "test_change_password": {
+        "password": "Verified@test"
+    }
+}

ux-tests/conftest.py

@@ -0,0 +1,134 @@
+import json
+
+import pytest
+from helpers import login
+from playwright.sync_api import sync_playwright
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "profile: run profile tests - requires --account to be set"
+    )
+    config.addinivalue_line("markers", "search: run search tests")
+    config.addinivalue_line("markers", "links: run follow links tests")
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--account",
+        action="store",
+        default="unauthenticated",
+        help="What type of user to run the tests as",
+    )
+
+    parser.addoption(
+        "--config",
+        action="store",
+        default="config.json",
+        help="Specify a config file to run the tests off of.",
+    )
+
+    parser.addoption(
+        "--browser",
+        action="store",
+        default="all",
+        help="Specify a browser to run the tests on.",
+        choices=("chromium", "firefox", "webkit", "all"),
+    )
+
+
+def pytest_runtest_setup(item):
+    if "profile" in item.keywords and "profile" not in item.config.option.markexpr:
+        pytest.skip("Test requires -m 'profile' to be set")
+
+    if "writes" in item.keywords and "writes" not in item.config.option.markexpr:
+        pytest.skip("Test requires -m 'writes' to be set")
+
+
+@pytest.fixture
+def account(request):
+    account_value = request.config.getoption("--account")
+    if account_value is None:
+        pytest.skip("Test requires --account to be set")
+    return account_value
+
+
+@pytest.fixture(scope="session")
+def config(request):
+    config_path = request.config.getoption("--config")
+
+    result = {}
+    with open(config_path) as config_file:
+        result = json.load(config_file)
+    return result
+
+
+@pytest.fixture(scope="session", params=["chromium", "firefox", "webkit"])
+def browser_type(request):
+    """
+    This fixture returns the type of the browser to use for the current test session.
+    """
+    # Use the --browser option to decide which browser to use
+    browser_option = request.config.getoption("--browser")
+    if browser_option == "all":
+        # If no specific browser is selected, return the current parameter
+        return request.param
+    elif request.param == browser_option:
+        # If a specific browser is selected, return it
+        return browser_option
+    else:
+        pytest.skip("Skipping tests for this browser")
+
+
+@pytest.fixture(scope="session")
+def account_credentials(config, request):
+    """
+    This fixture returns the credentials for the type of user specified in the --account option.
+    """
+    test_account = request.config.getoption("--account")
+    account_credentials = config["accounts"]
+    # if test account is found in credentials, return it
+    if test_account in account_credentials:
+        return account_credentials[test_account]
+    else:
+        return {}
+
+
+@pytest.fixture(scope="session")
+def page(request, config, browser_type):
+    """
+    This fixture creates a new browser context for each test session.
+    """
+    with sync_playwright() as p:
+        # Use the browser_type fixture to decide which browser to launch
+        if browser_type == "chromium":
+            browser = p.chromium.launch(headless=True)
+        elif browser_type == "firefox":
+            browser = p.firefox.launch(headless=True)
+        elif browser_type == "webkit":
+            browser = p.webkit.launch(headless=True)
+        else:
+            raise ValueError(f"Unsupported browser type: {browser_type}")
+
+        context = browser.new_context()
+        page = context.new_page()
+
+        page.goto(config["url"])
+        test_account = request.config.getoption("--account")
+        account_credentials = config["accounts"]
+
+        # if test account is found in credentials, login
+        if test_account in account_credentials:
+            login(
+                page,
+                account_credentials[test_account]["username"],
+                account_credentials[test_account]["password"],
+            )
+
+        page.goto(config["url"])
+        page.set_viewport_size({"width": 1920, "height": 1080})
+
+        yield page
+
+        context.close()
+        browser.close()

ux-tests/helpers.py

@@ -0,0 +1,21 @@
+from playwright.sync_api import Page
+
+
+def login(page: Page, username: str, password: str, retry=True):
+    """
+    This function logs into the website using the provided username and password.
+    """
+    page.click("text=Login")
+    page.fill("#id_auth-username", username)
+    page.fill("#id_auth-password", password)
+    page.click("form .btn-primary")
+
+    if retry:
+        try:
+            page.wait_for_selector(
+                'text="Please wait a bit before trying to login again."', timeout=5000
+            )
+            page.wait_for_timeout(30000)
+            login(page, username, password, retry=False)
+        except Exception:
+            pass

ux-tests/test_advanced_search.py

@@ -0,0 +1,154 @@
+import pytest
+from playwright.sync_api import Page
+
+TIMEOUT = 60000
+
+
+def get_id(category):
+    return {
+        "Exchanges": "ix",
+        "Networks": "net",
+        "Facilities": "fac",
+        "Organizations": "org",
+    }[category]
+
+
+def wait_for_results(page: Page, category):
+    """
+    This function waits for the search results to load on the page.
+    """
+    category_id = get_id(category)
+
+    # Wait for either the results to appear or for a "no results" message to appear
+    try:
+        page.wait_for_selector(f"#{category_id} .results div", timeout=TIMEOUT)
+    except Exception:
+        page.wait_for_selector(f"#{category_id} .results-empty", timeout=TIMEOUT)
+
+
+def advanced_search_for_name(page: Page, category, name):
+    """
+    This function performs an advanced search for the given name in the specified category.
+    """
+    category_id = get_id(category)
+    page.click('a[href="/advanced_search"]')
+    page.click(f'.advanced-search-view a[href="#{category_id}"]')
+    page.fill(
+        f'//div[@id="{category_id}"]//div[@data-edit-name="name_search"]//input',
+        name,
+        timeout=TIMEOUT,
+    )
+    page.click(f'//div[@id="{category_id}"]//a[@data-edit-action="submit"]')
+
+
+def check_advanced_search_results(page: Page, category, name):
+    """
+    This function checks if the advanced search results contain the expected name.
+    """
+    wait_for_results(page, category)
+
+    try:
+        page.wait_for_selector(
+            f'//div[@id="{get_id(category)}"]//div[@class="results"]'
+            + f'//a[@data-edit-name="name"][normalize-space()="{name}"]',
+            timeout=TIMEOUT,
+        )
+        return True
+    except Exception as exc:
+        return False, f"Element not found {exc}"
+
+
+@pytest.mark.search
+def test_advanced_search_exchange(config, page: Page):
+    """
+    This function tests the advanced search functionality for exchanges.
+    """
+    page.goto(config["url"])
+    ix_name = config["test_search_exchange"]["name"]
+    advanced_search_for_name(page, "Exchanges", ix_name)
+    assert check_advanced_search_results(page, "Exchanges", ix_name)
+
+
+@pytest.mark.search
+def test_advanced_search_network(config, page: Page):
+    """
+    This function tests the advanced search functionality for networks.
+    """
+    page.goto(config["url"])
+    network_name = config["test_search_network"]["name"]
+    advanced_search_for_name(page, "Networks", network_name)
+    assert check_advanced_search_results(page, "Networks", network_name)
+
+
+@pytest.mark.search
+def test_advanced_search_facility(config, page: Page):
+    """
+    This function tests the advanced search functionality for facilities.
+    """
+    page.goto(config["url"])
+    facility_name = config["test_search_facility"]["name"]
+    advanced_search_for_name(page, "Facilities", facility_name)
+    assert check_advanced_search_results(page, "Facilities", facility_name)
+
+
+@pytest.mark.search
+def test_advanced_search_organization(config, page: Page):
+    """
+    This function tests the advanced search functionality for organizations.
+    """
+    page.goto(config["url"])
+    org_name = config["test_search_organization"]["name"]
+    advanced_search_for_name(page, "Organizations", org_name)
+    assert check_advanced_search_results(page, "Organizations", org_name)
+
+
+@pytest.mark.search
+def test_advanced_search_url_exchange(config, page: Page):
+    """
+    This function tests the advanced search functionality for exchanges with URL checks.
+    """
+    page.goto(config["url"])
+    ix_name = config["test_search_exchange"]["name"]
+    advanced_search_for_name(page, "Exchanges", ix_name)
+    # reload with current url
+    page.goto(page.url)
+    assert check_advanced_search_results(page, "Exchanges", ix_name)
+
+
+@pytest.mark.search
+def test_advanced_search_url_network(config, page: Page):
+    """
+    This function tests the advanced search functionality for networks with URL checks.
+    """
+    page.goto(config["url"])
+    network_name = config["test_search_network"]["name"]
+    advanced_search_for_name(page, "Networks", network_name)
+    # reload with current url
+    page.goto(page.url)
+    assert check_advanced_search_results(page, "Networks", network_name)
+
+
+@pytest.mark.search
+def test_advanced_search_url_facility(config, page: Page):
+    """
+    This function tests the advanced search functionality for facilities with URL checks.
+    """
+    page.goto(config["url"])
+    facility_name = config["test_search_facility"]["name"]
+    advanced_search_for_name(page, "Facilities", facility_name)
+    # reload with current url
+    page.goto(page.url)
+    assert check_advanced_search_results(page, "Facilities", facility_name)
+
+
+@pytest.mark.search
+def test_advanced_search_url_organization(config, page: Page):
+    """
+    This function tests the advanced search functionality for organizations with URL checks.
+    """
+    page.goto(config["url"])
+    org_name = config["test_search_organization"]["name"]
+    advanced_search_for_name(page, "Organizations", org_name)
+    # reload with current url
+    page.goto(page.url)
+    assert check_advanced_search_results(page, "Organizations", org_name)

ux-tests/test_links.py

@@ -0,0 +1,37 @@
+import pytest
+from playwright.sync_api import Page
+
+
+@pytest.mark.links
+def test_links(config, page: Page, account_credentials):
+    """
+    This function tests all the links in the page.
+    """
+    page.goto(config["url"], wait_until="load")  # wait for the 'load' event
+    anchors = page.query_selector_all("a")
+    links = []
+    logout_link = None
+    for anchor in anchors:
+        link = page.evaluate("(el) => el.href", anchor)
+        if link and config["url"] in link:
+            if "logout" in link:
+                logout_link = link
+            else:
+                links.append(link)
+    # keep logout as the last
+    if logout_link:
+        links.append(logout_link)
+
+    # remove duplicates
+    links = list(dict.fromkeys(links))
+
+    # remove /docs
+    if config["url"] + "/docs" in links:
+        links.remove(config["url"] + "/docs")
+
+    for link in links:
+        page.goto(link, wait_until="load")  # wait for the 'load' event
+        try:
+            assert page.is_visible("#header .logo")
+        except Exception:
+            assert page.title() == "PeeringDB API Documentation"

ux-tests/test_profile.py

@@ -0,0 +1,118 @@
+import pytest
+from helpers import login
+from playwright.sync_api import Page
+
+TIMEOUT = 10000
+
+
+def change_password(page: Page, old_password, new_password):
+    """
+    This function changes the user's password.
+    """
+    page.fill(
+        '#form-change-password input[data-edit-name="password_c"]',
+        old_password,
+        timeout=TIMEOUT,
+    )
+    page.fill(
+        '#form-change-password input[data-edit-name="password"]',
+        new_password,
+        timeout=TIMEOUT,
+    )
+    page.fill(
+        '#form-change-password input[data-edit-name="password_v"]',
+        new_password,
+        timeout=TIMEOUT,
+    )
+    page.click(
+        '#form-change-password a.btn[data-edit-action="submit"]', timeout=TIMEOUT
+    )
+
+
+@pytest.mark.profile
+def test_profile_add_api_key(config, page: Page, account):
+    """
+    This function tests the functionality of adding an API key.
+    """
+    page.goto(config["url"] + "/profile")
+    num_of_keys = len(
+        page.query_selector_all('.api-keys div[data-edit-component="list"] div.row')
+    )
+
+    # add api key
+    page.fill(
+        '.api-keys div[data-edit-name="name"] input',
+        config["test_add_api_key"]["description"],
+        timeout=TIMEOUT,
+    )
+    page.click('.api-keys a[data-edit-action="add"]', timeout=TIMEOUT)
+
+    # check if api key added to list
+    assert page.wait_for_selector(
+        ".api-keys #api-key-popin-frame div.alert-success", timeout=TIMEOUT
+    )
+    assert num_of_keys + 1 == len(
+        page.query_selector_all('.api-keys div[data-edit-component="list"] div.row')
+    )
+
+
+@pytest.mark.profile
+def test_profile_delete_api_key(config, page: Page, account):
+    """
+    This function tests the functionality of deleting an API key.
+    """
+    # deletes an existing api key
+    page.goto(config["url"] + "/profile")
+    num_of_keys = len(
+        page.query_selector_all('.api-keys div[data-edit-component="list"] div.row')
+    )
+    page.on("dialog", lambda dialog: dialog.accept())
+
+    # remove key
+    description = config["test_delete_api_key"]["description"]
+    key_revoke_btn_selector = f'.api-keys div[data-edit-component="list"] div.row:has(span[data-edit-name="name"]:has-text("{description}")) a[data-edit-action="revoke"]'
+
+    page.wait_for_selector(key_revoke_btn_selector, timeout=TIMEOUT)
+    key_revoke_btn = page.query_selector(key_revoke_btn_selector)
+    key_revoke_btn.click()
+
+    page.goto(config["url"] + "/profile")
+
+    # check if key removed from list
+    assert num_of_keys - 1 == len(
+        page.query_selector_all('.api-keys div[data-edit-component="list"] div.row')
+    )
+
+
+@pytest.mark.profile
+def test_profile_change_password(config, page: Page, account_credentials, account):
+    """
+    This function tests the functionality of changing the user's password.
+    """
+    page.goto(config["url"] + "/profile", wait_until="load")
+
+    old_password = account_credentials["password"]
+    new_password = config["test_change_password"]["password"]
+    change_password(page, old_password, new_password)
+
+    # check for success message
+    assert page.wait_for_selector(
+        "#form-change-password #password-change-success", timeout=TIMEOUT
+    )
+
+    # re login
+    page.goto(config["url"], wait_until="load")
+    login(page, account_credentials["username"], new_password)
+
+    # change password back
+    page.goto(config["url"] + "/profile", wait_until="load")
+    change_password(page, new_password, old_password)
+
+    # check for success message
+    assert page.wait_for_selector(
+        "#form-change-password #password-change-success", timeout=TIMEOUT
+    )
+
+    # re login
+    page.goto(config["url"], wait_until="load")
+    login(page, account_credentials["username"], new_password)

ux-tests/test_search.py

@@ -0,0 +1,70 @@
+import pytest
+from playwright.sync_api import Page
+
+TIMEOUT = 60000
+
+
+def search_for_term(page: Page, term: str):
+    """
+    This function fills the search form with the given term and submits the form.
+    """
+    page.fill('form[action="/search"] input[id="search"]', term)
+    page.press('form[action="/search"] input[id="search"]', "Enter")
+
+
+def check_search_results(page: Page, category: str, term: str):
+    """
+    This function checks if the search results contain the given term in the specified category.
+    """
+    return page.wait_for_selector(
+        f'xpath=//div[@class="search-result"]//div[starts-with(., "{category}")]'
+        + f'//following-sibling::div//a[normalize-space()="{term}"]',
+        timeout=TIMEOUT,
+    )
+
+
+def test_search_exchange(config, page: Page):
+    """
+    This function tests the functionality of searching for an exchange.
+    """
+    page.goto(config["url"])
+    ix_name = config["test_search_exchange"]["name"]
+    search_for_term(page, ix_name)
+    assert check_search_results(page, "Exchanges", ix_name)
+
+
+@pytest.mark.search
+def test_search_network(config, page: Page):
+    """
+    This function tests the functionality of searching for a network.
+    """
+    page.goto(config["url"])
+    network_name = config["test_search_network"]["name"]
+    search_for_term(page, network_name)
+    assert check_search_results(
+        page,
+        "Networks",
+        config["test_search_network"].get("quick_search_result", network_name),
+    )
+
+
+@pytest.mark.search
+def test_search_facility(config, page: Page):
+    """
+    This function tests the functionality of searching for a facility.
+    """
+    page.goto(config["url"])
+    facility_name = config["test_search_facility"]["name"]
+    search_for_term(page, facility_name)
+    assert check_search_results(page, "Facilities", facility_name)
+
+
+@pytest.mark.search
+def test_search_organization(config, page: Page):
+    """
+    This function tests the functionality of searching for an organization.
+    """
+    page.goto(config["url"])
+    org_name = config["test_search_organization"]["name"]
+    search_for_term(page, org_name)
+    assert check_search_results(page, "Organizations", org_name)