net/hw/devlink_rate_tc_bw.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0

"""
Devlink Rate TC Bandwidth Test Suite
===================================

This test suite verifies the functionality of devlink-rate traffic class (TC)
bandwidth distribution in a virtualized environment. The tests validate that
bandwidth can be properly allocated between different traffic classes and
that TC mapping works as expected.

Test Environment:
----------------
- Creates 1 VF
- Establishes a bridge connecting the VF representor and the uplink representor
- Sets up 2 VLAN interfaces on the VF with different VLAN IDs (101, 102)
- Configures different traffic classes (TC3 and TC4) for each VLAN

Test Cases:
----------
1. test_no_tc_mapping_bandwidth:
   - Verifies that without TC mapping, bandwidth is NOT distributed according to
     the configured 80/20 split between TC4 and TC3
   - This test should fail if bandwidth matches the 80/20 split without TC
     mapping
   - Expected: Bandwidth should NOT be distributed as 80/20

2. test_tc_mapping_bandwidth:
   - Configures TC mapping using mqprio qdisc
   - Verifies that with TC mapping, bandwidth IS distributed according to the
     configured 80/20 split between TC3 and TC4
   - Expected: Bandwidth should be distributed as 80/20

Bandwidth Distribution:
----------------------
- TC3 (VLAN 101): Configured for 80% of total bandwidth
- TC4 (VLAN 102): Configured for 20% of total bandwidth
- Total bandwidth: 1Gbps
- Tolerance: +-12%

Hardware-Specific Behavior (mlx5):
--------------------------
mlx5 hardware enforces traffic class separation by ensuring that each transmit
queue (SQ) is associated with a single TC. If a packet is sent on a queue that
doesn't match the expected TC (based on DSCP or VLAN priority and hypervisor-set
mapping), the hardware moves the queue to the correct TC scheduler to preserve
traffic isolation.

This behavior means that even without explicit TC-to-queue mapping, bandwidth
enforcement may still appear to work—because the hardware dynamically adjusts
the scheduling context. However, this can lead to performance issues in high
rates and HOL blocking if traffic from different TCs is mixed on the same queue.
"""

import json
import os
import subprocess
import threading
import time

from lib.py import ksft_pr, ksft_run, ksft_exit
from lib.py import KsftSkipEx, KsftFailEx, KsftXfailEx
from lib.py import NetDrvEpEnv, DevlinkFamily
from lib.py import NlError
from lib.py import cmd, defer, ethtool, ip


class BandwidthValidator:
    """
    Validates bandwidth totals and per-TC shares against expected values
    with a tolerance.
    """

    def __init__(self):
        self.tolerance_percent = 12
        self.expected_total_gbps = 1.0
        self.total_min_expected = self.min_expected(self.expected_total_gbps)
        self.total_max_expected = self.max_expected(self.expected_total_gbps)
        self.tc_expected_percent = {
            3: 20.0,
            4: 80.0,
        }

    def min_expected(self, value):
        """Calculates the minimum acceptable value based on tolerance."""
        return value - (value * self.tolerance_percent / 100)

    def max_expected(self, value):
        """Calculates the maximum acceptable value based on tolerance."""
        return value + (value * self.tolerance_percent / 100)

    def bound(self, expected, value):
        """Returns True if value is within expected tolerance."""
        return self.min_expected(expected) <= value <= self.max_expected(expected)

    def tc_bandwidth_bound(self, value, tc_ix):
        """
        Returns True if the given bandwidth value is within tolerance
        for the TC's expected bandwidth.
        """
        expected = self.tc_expected_percent[tc_ix]
        return self.bound(expected, value)


def setup_vf(cfg, set_tc_mapping=True):
    """
    Sets up a VF on the given network interface.

    Enables SR-IOV and switchdev mode, brings the VF interface up,
    and optionally configures TC mapping using mqprio.
    """
    try:
        cmd(f"devlink dev eswitch set pci/{cfg.pci} mode switchdev")
        defer(cmd, f"devlink dev eswitch set pci/{cfg.pci} mode legacy")
    except Exception as exc:
        raise KsftSkipEx(f"Failed to enable switchdev mode on {cfg.pci}") from exc
    try:
        cmd(f"echo 1 > /sys/class/net/{cfg.ifname}/device/sriov_numvfs")
        defer(cmd, f"echo 0 > /sys/class/net/{cfg.ifname}/device/sriov_numvfs")
    except Exception as exc:
        raise KsftSkipEx(f"Failed to enable SR-IOV on {cfg.ifname}") from exc

    time.sleep(2)
    vf_ifc = (os.listdir(
        f"/sys/class/net/{cfg.ifname}/device/virtfn0/net") or [None])[0]
    if vf_ifc:
        ip(f"link set dev {vf_ifc} up")
    else:
        raise KsftSkipEx("VF interface not found")
    if set_tc_mapping:
        cmd(f"tc qdisc add dev {vf_ifc} root handle 5 mqprio mode dcb hw 1 num_tc 8")

    return vf_ifc


def setup_vlans_on_vf(vf_ifc):
    """
    Sets up two VLAN interfaces on the given VF, each mapped to a different TC.
    """
    vlan_configs = [
        {"vlan_id": 101, "tc": 3, "ip": "198.51.100.2"},
        {"vlan_id": 102, "tc": 4, "ip": "198.51.100.10"},
    ]

    for config in vlan_configs:
        vlan_dev = f"{vf_ifc}.{config['vlan_id']}"
        ip(f"link add link {vf_ifc} name {vlan_dev} type vlan id {config['vlan_id']}")
        ip(f"addr add {config['ip']}/29 dev {vlan_dev}")
        ip(f"link set dev {vlan_dev} up")
        ip(f"link set dev {vlan_dev} type vlan egress-qos-map 0:{config['tc']}")
        ksft_pr(f"Created VLAN {vlan_dev} on {vf_ifc} with tc {config['tc']} and IP {config['ip']}")


def get_vf_info(cfg):
    """
    Finds the VF representor interface and devlink port index
    for the given PCI device used in the test environment.
    """
    cfg.vf_representor = None
    cfg.vf_port_index = None
    out = subprocess.check_output(["devlink", "-j", "port", "show"], encoding="utf-8")
    ports = json.loads(out)["port"]

    for port_name, props in ports.items():
        netdev = props.get("netdev")

        if (port_name.startswith(f"pci/{cfg.pci}/") and
            props.get("vfnum") == 0):
            cfg.vf_representor = netdev
            cfg.vf_port_index = int(port_name.split("/")[-1])
            break


def setup_bridge(cfg):
    """
    Creates and configures a Linux bridge, with both the uplink
    and VF representor interfaces attached to it.
    """
    bridge_name = f"br_{os.getpid()}"
    ip(f"link add name {bridge_name} type bridge")
    defer(cmd, f"ip link del name {bridge_name} type bridge")

    ip(f"link set dev {cfg.ifname} master {bridge_name}")

    rep_name = cfg.vf_representor
    if rep_name:
        ip(f"link set dev {rep_name} master {bridge_name}")
        ip(f"link set dev {rep_name} up")
        ksft_pr(f"Set representor {rep_name} up and added to bridge")
    else:
        raise KsftSkipEx("Could not find representor for the VF")

    ip(f"link set dev {bridge_name} up")


def setup_devlink_rate(cfg):
    """
    Configures devlink rate tx_max and traffic class bandwidth for the VF.
    """
    port_index = cfg.vf_port_index
    if port_index is None:
        raise KsftSkipEx("Could not find VF port index")
    try:
        cfg.devnl.rate_set({
            "bus-name": "pci",
            "dev-name": cfg.pci,
            "port-index": port_index,
            "rate-tx-max": 125000000,
            "rate-tc-bws": [
                {"index": 0, "bw": 0},
                {"index": 1, "bw": 0},
                {"index": 2, "bw": 0},
                {"index": 3, "bw": 20},
                {"index": 4, "bw": 80},
                {"index": 5, "bw": 0},
                {"index": 6, "bw": 0},
                {"index": 7, "bw": 0},
            ]
        })
    except NlError as exc:
        if exc.error == 95:  # EOPNOTSUPP
            raise KsftSkipEx("devlink rate configuration is not supported on the VF") from exc
        raise KsftFailEx(f"rate_set failed on VF port {port_index}") from exc


def setup_remote_server(cfg):
    """
    Sets up VLAN interfaces and starts iperf3 servers on the remote side.
    """
    remote_dev = cfg.remote_ifname
    vlan_ids = [101, 102]
    remote_ips = ["198.51.100.1", "198.51.100.9"]

    for vlan_id, ip_addr in zip(vlan_ids, remote_ips):
        vlan_dev = f"{remote_dev}.{vlan_id}"
        cmd(f"ip link add link {remote_dev} name {vlan_dev} "
            f"type vlan id {vlan_id}", host=cfg.remote)
        cmd(f"ip addr add {ip_addr}/29 dev {vlan_dev}", host=cfg.remote)
        cmd(f"ip link set dev {vlan_dev} up", host=cfg.remote)
        cmd(f"iperf3 -s -1 -B {ip_addr}",background=True, host=cfg.remote)
        defer(cmd, f"ip link del {vlan_dev}", host=cfg.remote)


def setup_test_environment(cfg, set_tc_mapping=True):
    """
    Sets up the complete test environment including VF creation, VLANs,
    bridge configuration, devlink rate setup, and the remote server.
    """
    vf_ifc = setup_vf(cfg, set_tc_mapping)
    ksft_pr(f"Created VF interface: {vf_ifc}")

    setup_vlans_on_vf(vf_ifc)

    get_vf_info(cfg)
    setup_bridge(cfg)

    setup_devlink_rate(cfg)
    setup_remote_server(cfg)
    time.sleep(2)


def run_iperf_client(server_ip, local_ip, barrier, min_expected_gbps=0.1):
    """
    Runs a single iperf3 client instance, binding to the given local IP.
    Waits on a barrier to synchronize with other threads.
    """
    try:
        barrier.wait(timeout=10)
    except Exception as exc:
        raise KsftFailEx("iperf3 barrier wait timed") from exc

    iperf_cmd = ["iperf3", "-c", server_ip, "-B", local_ip, "-J"]
    result = subprocess.run(iperf_cmd, capture_output=True, text=True,
                            check=True)

    try:
        output = json.loads(result.stdout)
        bits_per_second = output["end"]["sum_received"]["bits_per_second"]
        gbps = bits_per_second / 1e9
        if gbps < min_expected_gbps:
            ksft_pr(
                f"iperf3 bandwidth too low: {gbps:.2f} Gbps "
                f"(expected ≥ {min_expected_gbps} Gbps)"
            )
            return None
        return gbps
    except json.JSONDecodeError as exc:
        ksft_pr(f"Failed to parse iperf3 JSON output: {exc}")
        return None


def run_bandwidth_test():
    """
    Launches iperf3 client threads for each VLAN/TC pair and collects results.
    """
    def _run_iperf_client_thread(server_ip, local_ip, results, barrier, tc_ix):
        results[tc_ix] = run_iperf_client(server_ip, local_ip, barrier)

    vf_vlan_data = [
        # (local_ip, remote_ip, TC)
        ("198.51.100.2",  "198.51.100.1", 3),
        ("198.51.100.10", "198.51.100.9", 4),
    ]

    results = {}
    threads = []
    start_barrier = threading.Barrier(len(vf_vlan_data))

    for local_ip, remote_ip, tc_ix in vf_vlan_data:
        thread = threading.Thread(
            target=_run_iperf_client_thread,
            args=(remote_ip, local_ip, results, start_barrier, tc_ix)
        )
        thread.start()
        threads.append(thread)

    for thread in threads:
        thread.join()

    for tc_ix, tc_bw in results.items():
        if tc_bw is None:
            raise KsftFailEx("iperf3 client failed; cannot evaluate bandwidth")

    return results

def calculate_bandwidth_percentages(results):
    """
    Calculates the percentage of total bandwidth received by TC3 and TC4.
    """
    if 3 not in results or 4 not in results:
        raise KsftFailEx(f"Missing expected TC results in {results}")

    tc3_bw = results[3]
    tc4_bw = results[4]
    total_bw = tc3_bw + tc4_bw
    tc3_percentage = (tc3_bw / total_bw) * 100
    tc4_percentage = (tc4_bw / total_bw) * 100

    return {
        'tc3_bw': tc3_bw,
        'tc4_bw': tc4_bw,
        'tc3_percentage': tc3_percentage,
        'tc4_percentage': tc4_percentage,
        'total_bw': total_bw
    }


def print_bandwidth_results(bw_data, test_name):
    """
    Prints bandwidth measurements and TC usage summary for a given test.
    """
    ksft_pr(f"Bandwidth check results {test_name}:")
    ksft_pr(f"TC 3: {bw_data['tc3_bw']:.2f} Gbits/sec")
    ksft_pr(f"TC 4: {bw_data['tc4_bw']:.2f} Gbits/sec")
    ksft_pr(f"Total bandwidth: {bw_data['total_bw']:.2f} Gbits/sec")
    ksft_pr(f"TC 3 percentage: {bw_data['tc3_percentage']:.1f}%")
    ksft_pr(f"TC 4 percentage: {bw_data['tc4_percentage']:.1f}%")


def verify_total_bandwidth(bw_data, validator):
    """
    Ensures the total measured bandwidth falls within the acceptable tolerance.
    """
    total = bw_data['total_bw']

    if validator.bound(validator.expected_total_gbps, total):
        return

    if total < validator.total_min_expected:
        raise KsftSkipEx(
            f"Total bandwidth {total:.2f} Gbps < minimum "
            f"{validator.total_min_expected:.2f} Gbps; "
            f"parent tx_max ({validator.expected_total_gbps:.1f} G) "
            f"not reached, cannot validate share"
        )

    raise KsftFailEx(
        f"Total bandwidth {total:.2f} Gbps exceeds allowed ceiling "
        f"{validator.total_max_expected:.2f} Gbps "
        f"(VF tx_max set to {validator.expected_total_gbps:.1f} G)"
    )


def check_bandwidth_distribution(bw_data, validator):
    """
    Checks whether the measured TC3 and TC4 bandwidth percentages
    fall within their expected tolerance ranges.

    Returns:
        bool: True if both TC3 and TC4 percentages are within bounds.
    """
    tc3_valid = validator.tc_bandwidth_bound(bw_data['tc3_percentage'], 3)
    tc4_valid = validator.tc_bandwidth_bound(bw_data['tc4_percentage'], 4)

    return tc3_valid and tc4_valid


def run_bandwidth_distribution_test(cfg, set_tc_mapping):
    """
    Runs parallel iperf3 tests for both TCs and collects results.
    """
    setup_test_environment(cfg, set_tc_mapping)
    bandwidths = run_bandwidth_test()
    bw_data = calculate_bandwidth_percentages(bandwidths)
    test_name = "with TC mapping" if set_tc_mapping else "without TC mapping"
    print_bandwidth_results(bw_data, test_name)

    verify_total_bandwidth(bw_data, cfg.bw_validator)

    return check_bandwidth_distribution(bw_data, cfg.bw_validator)


def test_no_tc_mapping_bandwidth(cfg):
    """
    Verifies that bandwidth is not split 80/20 without traffic class mapping.
    """
    pass_bw_msg = "Bandwidth is NOT distributed as 80/20 without TC mapping"
    fail_bw_msg = "Bandwidth matched 80/20 split without TC mapping"
    is_mlx5 = "driver: mlx5" in ethtool(f"-i {cfg.ifname}").stdout

    if run_bandwidth_distribution_test(cfg, set_tc_mapping=False):
        if is_mlx5:
            raise KsftXfailEx(fail_bw_msg)
        raise KsftFailEx(fail_bw_msg)
    if is_mlx5:
        raise KsftFailEx("mlx5 behavior changed:" + pass_bw_msg)
    ksft_pr(pass_bw_msg)


def test_tc_mapping_bandwidth(cfg):
    """
    Verifies that bandwidth is correctly split 80/20 between TC3 and TC4
    when traffic class mapping is set.
    """
    if run_bandwidth_distribution_test(cfg, set_tc_mapping=True):
        ksft_pr("Bandwidth is distributed as 80/20 with TC mapping")
    else:
        raise KsftFailEx("Bandwidth did not match 80/20 split with TC mapping")


def main() -> None:
    """
    Main entry point for running the test cases.
    """
    with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
        cfg.devnl = DevlinkFamily()

        cfg.pci = os.path.basename(
            os.path.realpath(f"/sys/class/net/{cfg.ifname}/device")
        )
        if not cfg.pci:
            raise KsftSkipEx("Could not get PCI address of the interface")
        cfg.require_cmd("iperf3", local=True, remote=True)

        cfg.bw_validator = BandwidthValidator()

        cases = [test_no_tc_mapping_bandwidth, test_tc_mapping_bandwidth]

        ksft_run(cases=cases, args=(cfg,))
    ksft_exit()


if __name__ == "__main__":
    main()