1import os
2import re
3
4import infra.basetest
5
6
7class TestNumaCtl(infra.basetest.BRTest):
8    # A specific configuration is needed for testing numactl:
9    # - This test uses a x86_64 config, which has mature NUMA support.
10    # - A kernel need to compiled with a NUMA support.
11    kernel_fragment = \
12        infra.filepath("tests/package/test_numactl/linux-numactl.fragment")
13    config = \
14        f"""
15        BR2_x86_64=y
16        BR2_x86_corei7=y
17        BR2_TOOLCHAIN_EXTERNAL=y
18        BR2_LINUX_KERNEL=y
19        BR2_LINUX_KERNEL_CUSTOM_VERSION=y
20        BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE="6.1.75"
21        BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y
22        BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="board/qemu/x86_64/linux.config"
23        BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{kernel_fragment}"
24        BR2_LINUX_KERNEL_NEEDS_HOST_OPENSSL=y
25        BR2_LINUX_KERNEL_NEEDS_HOST_LIBELF=y
26        BR2_PACKAGE_NUMACTL=y
27        BR2_TARGET_ROOTFS_CPIO=y
28        BR2_TARGET_ROOTFS_CPIO_GZIP=y
29        # BR2_TARGET_ROOTFS_TAR is not set
30        """
31
32    def check_numactl_preferred(self):
33        # Show the default NUMA policy settings. We check we have the
34        # 4 physical cpus on 2 nodes we configured the emulator
35        # command line.
36        out, ret = self.emulator.run("numactl --show")
37        self.assertEqual(ret, 0)
38        checks = [
39            "policy: default",
40            "preferred node: current",
41            "physcpubind: 0 1 2 3 ",
42            "nodebind: 0 1 ",
43            "membind: 0 1 "
44        ]
45        for pattern in checks:
46            self.assertIn(pattern, out)
47
48        # Check the preferred policy on different nodes. This command
49        # is taken from the numactl man page.
50        for pref_node in range(2):
51            cmd = f"numactl --preferred={pref_node} numactl --show"
52            out, ret = self.emulator.run(cmd)
53            self.assertEqual(ret, 0)
54            checks = [
55                "policy: preferred",
56                f"preferred node: {pref_node}"
57            ]
58            for pattern in checks:
59                self.assertIn(pattern, out)
60
61    def get_numa_node_free_mem(self):
62        out, ret = self.emulator.run("numactl --hardware")
63        self.assertEqual(ret, 0)
64        free_mem = {}
65        p = re.compile("^node ([0-9]+) free: ([0-9]+) MB")
66        for line in out:
67            m = p.match(line)
68            if m:
69                node = int(m.group(1))
70                mem = int(m.group(2))
71                free_mem[node] = mem
72        return free_mem
73
74    def check_numactl_membind(self):
75        # We get the current amount of free memory on each node, for
76        # later comparison.
77        initial_node_free_mem = self.get_numa_node_free_mem()
78
79        # We allocate a shared memory file with a restriction to be in
80        # node 1 memory only.
81        shm_file = "/dev/shm/file"
82        file_size = 100
83        cmd = f"numactl --membind=1 dd if=/dev/zero of={shm_file} bs=1M count={file_size}"
84        self.assertRunOk(cmd)
85
86        # We collect again the amount of free memory per node.
87        node_free_mem = self.get_numa_node_free_mem()
88
89        # Since we allocated 100M on node 1 only, we check the free
90        # space on node 0 did not significantly changed and on node 1
91        # approximately reduced of the file size.
92        diff = initial_node_free_mem[0] - node_free_mem[0]
93        self.assertAlmostEqual(diff, 0, delta=10)
94        diff = initial_node_free_mem[1] - node_free_mem[1]
95        self.assertAlmostEqual(diff, file_size, delta=10)
96
97        # Remove the file, to free the memory.
98        self.assertRunOk(f"rm -f {shm_file}")
99
100        # We allocate again a file in shared memory, but this time in
101        # two chunks. Each chunk is requested to be allocated in two
102        # different nodes. This example is taken from the numactl man
103        # page.
104        chunk_size = file_size // 2
105        cmd = "numactl --membind=0 "
106        cmd += f"dd if=/dev/zero of={shm_file} bs=1M count={chunk_size}"
107        self.assertRunOk(cmd)
108        cmd = "numactl --membind=1 "
109        cmd += f"dd if=/dev/zero of={shm_file} bs=1M count={chunk_size} seek={chunk_size}"
110        self.assertRunOk(cmd)
111
112        # We collect again the amount of free memory.
113        node_free_mem = self.get_numa_node_free_mem()
114
115        # We check the free memory space approximately reduced of each
116        # chunk size.
117        for node in range(2):
118            free_mem_diff = initial_node_free_mem[node] - node_free_mem[node]
119            self.assertAlmostEqual(free_mem_diff, chunk_size, delta=5)
120
121    def test_run(self):
122        img = os.path.join(self.builddir, "images", "rootfs.cpio.gz")
123        kern = os.path.join(self.builddir, "images", "bzImage")
124        # We start the Qemu emulator with 4 processors on 2 NUMA nodes.
125        self.emulator.boot(arch="x86_64",
126                           kernel=kern,
127                           kernel_cmdline=["console=ttyS0"],
128                           options=["-cpu", "Nehalem", "-m", "512M",
129                                    "-smp", "cpus=4,sockets=2,cores=2,maxcpus=4",
130                                    "-object", "memory-backend-ram,size=256M,id=m0",
131                                    "-object", "memory-backend-ram,size=256M,id=m1",
132                                    "-numa", "node,cpus=0-1,nodeid=0,memdev=m0",
133                                    "-numa", "node,cpus=2-3,nodeid=1,memdev=m1",
134                                    "-initrd", img])
135        self.emulator.login()
136
137        # Check a simple numactl invication:
138        # show the NUMA hardware inventory.
139        self.assertRunOk("numactl --hardware")
140
141        self.check_numactl_preferred()
142        self.check_numactl_membind()
143