1#!/usr/bin/env python3
2
3# Copyright (C) 2017 Netronome Systems, Inc.
4# Copyright (c) 2019 Mellanox Technologies. All rights reserved
5#
6# This software is licensed under the GNU General License Version 2,
7# June 1991 as shown in the file COPYING in the top-level directory of this
8# source tree.
9#
10# THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
11# WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
12# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
13# FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
14# OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
15# THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16
17from datetime import datetime
18import argparse
19import errno
20import json
21import os
22import pprint
23import random
24import re
25import stat
26import string
27import struct
28import subprocess
29import time
30import traceback
31
32from lib.py import NetdevSim, NetdevSimDev
33
34
35logfile = None
36log_level = 1
37skip_extack = False
38bpf_test_dir = os.path.dirname(os.path.realpath(__file__))
39pp = pprint.PrettyPrinter()
40devs = [] # devices we created for clean up
41files = [] # files to be removed
42netns = [] # net namespaces to be removed
43
44def log_get_sec(level=0):
45    return "*" * (log_level + level)
46
47def log_level_inc(add=1):
48    global log_level
49    log_level += add
50
51def log_level_dec(sub=1):
52    global log_level
53    log_level -= sub
54
55def log_level_set(level):
56    global log_level
57    log_level = level
58
59def log(header, data, level=None):
60    """
61    Output to an optional log.
62    """
63    if logfile is None:
64        return
65    if level is not None:
66        log_level_set(level)
67
68    if not isinstance(data, str):
69        data = pp.pformat(data)
70
71    if len(header):
72        logfile.write("\n" + log_get_sec() + " ")
73        logfile.write(header)
74    if len(header) and len(data.strip()):
75        logfile.write("\n")
76    logfile.write(data)
77
78def skip(cond, msg):
79    if not cond:
80        return
81    print("SKIP: " + msg)
82    log("SKIP: " + msg, "", level=1)
83    os.sys.exit(0)
84
85def fail(cond, msg):
86    if not cond:
87        return
88    print("FAIL: " + msg)
89    tb = "".join(traceback.extract_stack().format())
90    print(tb)
91    log("FAIL: " + msg, tb, level=1)
92    os.sys.exit(1)
93
94def start_test(msg):
95    log(msg, "", level=1)
96    log_level_inc()
97    print(msg)
98
99def cmd(cmd, shell=True, include_stderr=False, background=False, fail=True):
100    """
101    Run a command in subprocess and return tuple of (retval, stdout);
102    optionally return stderr as well as third value.
103    """
104    proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE,
105                            stderr=subprocess.PIPE)
106    if background:
107        msg = "%s START: %s" % (log_get_sec(1),
108                                datetime.now().strftime("%H:%M:%S.%f"))
109        log("BKG " + proc.args, msg)
110        return proc
111
112    return cmd_result(proc, include_stderr=include_stderr, fail=fail)
113
114def cmd_result(proc, include_stderr=False, fail=False):
115    stdout, stderr = proc.communicate()
116    stdout = stdout.decode("utf-8")
117    stderr = stderr.decode("utf-8")
118    proc.stdout.close()
119    proc.stderr.close()
120
121    stderr = "\n" + stderr
122    if stderr[-1] == "\n":
123        stderr = stderr[:-1]
124
125    sec = log_get_sec(1)
126    log("CMD " + proc.args,
127        "RETCODE: %d\n%s STDOUT:\n%s%s STDERR:%s\n%s END: %s" %
128        (proc.returncode, sec, stdout, sec, stderr,
129         sec, datetime.now().strftime("%H:%M:%S.%f")))
130
131    if proc.returncode != 0 and fail:
132        if len(stderr) > 0 and stderr[-1] == "\n":
133            stderr = stderr[:-1]
134        raise Exception("Command failed: %s\n%s" % (proc.args, stderr))
135
136    if include_stderr:
137        return proc.returncode, stdout, stderr
138    else:
139        return proc.returncode, stdout
140
141def rm(f):
142    cmd("rm -f %s" % (f))
143    if f in files:
144        files.remove(f)
145
146def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False):
147    params = ""
148    if JSON:
149        params += "%s " % (flags["json"])
150
151    if ns:
152        ns = "ip netns exec %s " % (ns)
153    elif ns is None:
154        ns = ""
155
156    if include_stderr:
157        ret, stdout, stderr = cmd(ns + name + " " + params + args,
158                                  fail=fail, include_stderr=True)
159    else:
160        ret, stdout = cmd(ns + name + " " + params + args,
161                          fail=fail, include_stderr=False)
162
163    if JSON and len(stdout.strip()) != 0:
164        out = json.loads(stdout)
165    else:
166        out = stdout
167
168    if include_stderr:
169        return ret, out, stderr
170    else:
171        return ret, out
172
173def bpftool(args, JSON=True, ns="", fail=True, include_stderr=False):
174    return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns,
175                fail=fail, include_stderr=include_stderr)
176
177def bpftool_prog_list(expected=None, ns="", exclude_orphaned=True):
178    _, progs = bpftool("prog show", JSON=True, ns=ns, fail=True)
179    # Remove the base progs
180    for p in base_progs:
181        if p in progs:
182            progs.remove(p)
183    if exclude_orphaned:
184        progs = [ p for p in progs if not p['orphaned'] ]
185    if expected is not None:
186        if len(progs) != expected:
187            fail(True, "%d BPF programs loaded, expected %d" %
188                 (len(progs), expected))
189    return progs
190
191def bpftool_map_list(expected=None, ns=""):
192    _, maps = bpftool("map show", JSON=True, ns=ns, fail=True)
193    # Remove the base maps
194    maps = [m for m in maps if m not in base_maps and m.get('name') and m.get('name') not in base_map_names]
195    if expected is not None:
196        if len(maps) != expected:
197            fail(True, "%d BPF maps loaded, expected %d" %
198                 (len(maps), expected))
199    return maps
200
201def bpftool_prog_list_wait(expected=0, n_retry=20):
202    for i in range(n_retry):
203        nprogs = len(bpftool_prog_list())
204        if nprogs == expected:
205            return
206        time.sleep(0.05)
207    raise Exception("Time out waiting for program counts to stabilize want %d, have %d" % (expected, nprogs))
208
209def bpftool_map_list_wait(expected=0, n_retry=20, ns=""):
210    for i in range(n_retry):
211        maps = bpftool_map_list(ns=ns)
212        if len(maps) == expected:
213            return maps
214        time.sleep(0.05)
215    raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps))
216
217def bpftool_prog_load(sample, file_name, maps=[], prog_type="xdp", dev=None,
218                      fail=True, include_stderr=False):
219    args = "prog load %s %s" % (os.path.join(bpf_test_dir, sample), file_name)
220    if prog_type is not None:
221        args += " type " + prog_type
222    if dev is not None:
223        args += " dev " + dev
224    if len(maps):
225        args += " map " + " map ".join(maps)
226
227    res = bpftool(args, fail=fail, include_stderr=include_stderr)
228    if res[0] == 0:
229        files.append(file_name)
230    return res
231
232def ip(args, force=False, JSON=True, ns="", fail=True, include_stderr=False):
233    if force:
234        args = "-force " + args
235    return tool("ip", args, {"json":"-j"}, JSON=JSON, ns=ns,
236                fail=fail, include_stderr=include_stderr)
237
238def tc(args, JSON=True, ns="", fail=True, include_stderr=False):
239    return tool("tc", args, {"json":"-p"}, JSON=JSON, ns=ns,
240                fail=fail, include_stderr=include_stderr)
241
242def ethtool(dev, opt, args, fail=True):
243    return cmd("ethtool %s %s %s" % (opt, dev["ifname"], args), fail=fail)
244
245def bpf_obj(name, sec="xdp", path=bpf_test_dir,):
246    return "obj %s sec %s" % (os.path.join(path, name), sec)
247
248def bpf_pinned(name):
249    return "pinned %s" % (name)
250
251def bpf_bytecode(bytecode):
252    return "bytecode \"%s\"" % (bytecode)
253
254def mknetns(n_retry=10):
255    for i in range(n_retry):
256        name = ''.join([random.choice(string.ascii_letters) for i in range(8)])
257        ret, _ = ip("netns add %s" % (name), fail=False)
258        if ret == 0:
259            netns.append(name)
260            return name
261    return None
262
263def int2str(fmt, val):
264    ret = []
265    for b in struct.pack(fmt, val):
266        ret.append(int(b))
267    return " ".join(map(lambda x: str(x), ret))
268
269def str2int(strtab):
270    inttab = []
271    for i in strtab:
272        inttab.append(int(i, 16))
273    ba = bytearray(inttab)
274    if len(strtab) == 4:
275        fmt = "I"
276    elif len(strtab) == 8:
277        fmt = "Q"
278    else:
279        raise Exception("String array of len %d can't be unpacked to an int" %
280                        (len(strtab)))
281    return struct.unpack(fmt, ba)[0]
282
283class DebugfsDir:
284    """
285    Class for accessing DebugFS directories as a dictionary.
286    """
287
288    def __init__(self, path):
289        self.path = path
290        self._dict = self._debugfs_dir_read(path)
291
292    def __len__(self):
293        return len(self._dict.keys())
294
295    def __getitem__(self, key):
296        if type(key) is int:
297            key = list(self._dict.keys())[key]
298        return self._dict[key]
299
300    def __setitem__(self, key, value):
301        log("DebugFS set %s = %s" % (key, value), "")
302        log_level_inc()
303
304        cmd("echo '%s' > %s/%s" % (value, self.path, key))
305        log_level_dec()
306
307        _, out = cmd('cat %s/%s' % (self.path, key))
308        self._dict[key] = out.strip()
309
310    def _debugfs_dir_read(self, path):
311        dfs = {}
312
313        log("DebugFS state for %s" % (path), "")
314        log_level_inc(add=2)
315
316        _, out = cmd('ls ' + path)
317        for f in out.split():
318            if f == "ports":
319                continue
320
321            p = os.path.join(path, f)
322            if not os.stat(p).st_mode & stat.S_IRUSR:
323                continue
324
325            if os.path.isfile(p):
326                # We need to init trap_flow_action_cookie before read it
327                if f == "trap_flow_action_cookie":
328                    cmd('echo deadbeef > %s/%s' % (path, f))
329                _, out = cmd('cat %s/%s' % (path, f))
330                dfs[f] = out.strip()
331            elif os.path.isdir(p):
332                dfs[f] = DebugfsDir(p)
333            else:
334                raise Exception("%s is neither file nor directory" % (p))
335
336        log_level_dec()
337        log("DebugFS state", dfs)
338        log_level_dec()
339
340        return dfs
341
342class BpfNetdevSimDev(NetdevSimDev):
343    """
344    Class for netdevsim bus device and its attributes.
345    """
346    def __init__(self, port_count=1, ns=None):
347        super().__init__(port_count, ns=ns)
348        devs.append(self)
349
350    def _make_port(self, port_index, ifname):
351        return BpfNetdevSim(self, port_index, ifname, self.ns)
352
353    def dfs_num_bound_progs(self):
354        path = os.path.join(self.dfs_dir, "bpf_bound_progs")
355        _, progs = cmd('ls %s' % (path))
356        return len(progs.split())
357
358    def dfs_get_bound_progs(self, expected):
359        progs = DebugfsDir(os.path.join(self.dfs_dir, "bpf_bound_progs"))
360        if expected is not None:
361            if len(progs) != expected:
362                fail(True, "%d BPF programs bound, expected %d" %
363                     (len(progs), expected))
364        return progs
365
366    def remove(self):
367        super().remove()
368        devs.remove(self)
369
370
371class BpfNetdevSim(NetdevSim):
372    """
373    Class for netdevsim netdevice and its attributes.
374    """
375
376    def __init__(self, nsimdev, port_index, ifname, ns=None):
377        super().__init__(nsimdev, port_index, ifname, ns=ns)
378
379        self.dfs_dir = "%s/ports/%u/" % (nsimdev.dfs_dir, port_index)
380        self.dfs_refresh()
381
382    def __getitem__(self, key):
383        return self.dev[key]
384
385    def remove(self):
386        self.nsimdev.remove_nsim(self)
387
388    def dfs_refresh(self):
389        self.dfs = DebugfsDir(self.dfs_dir)
390        return self.dfs
391
392    def dfs_read(self, f):
393        path = os.path.join(self.dfs_dir, f)
394        _, data = cmd('cat %s' % (path))
395        return data.strip()
396
397    def wait_for_flush(self, bound=0, total=0, n_retry=20):
398        for i in range(n_retry):
399            nbound = self.nsimdev.dfs_num_bound_progs()
400            nprogs = len(bpftool_prog_list())
401            if nbound == bound and nprogs == total:
402                return
403            time.sleep(0.05)
404        raise Exception("Time out waiting for program counts to stabilize want %d/%d, have %d bound, %d loaded" % (bound, total, nbound, nprogs))
405
406    def set_ns(self, ns):
407        name = ns if ns else "1"
408        ip("link set dev %s netns %s" % (self.dev["ifname"], name), ns=self.ns)
409        self.ns = ns
410
411    def set_mtu(self, mtu, fail=True):
412        return ip("link set dev %s mtu %d" % (self.dev["ifname"], mtu),
413                  fail=fail)
414
415    def set_xdp(self, bpf, mode, force=False, JSON=True, verbose=False,
416                fail=True, include_stderr=False):
417        if verbose:
418            bpf += " verbose"
419        return ip("link set dev %s xdp%s %s" % (self.dev["ifname"], mode, bpf),
420                  force=force, JSON=JSON,
421                  fail=fail, include_stderr=include_stderr)
422
423    def unset_xdp(self, mode, force=False, JSON=True,
424                  fail=True, include_stderr=False):
425        return ip("link set dev %s xdp%s off" % (self.dev["ifname"], mode),
426                  force=force, JSON=JSON,
427                  fail=fail, include_stderr=include_stderr)
428
429    def ip_link_show(self, xdp):
430        _, link = ip("link show dev %s" % (self['ifname']))
431        if len(link) > 1:
432            raise Exception("Multiple objects on ip link show")
433        if len(link) < 1:
434            return {}
435        fail(xdp != "xdp" in link,
436             "XDP program not reporting in iplink (reported %s, expected %s)" %
437             ("xdp" in link, xdp))
438        return link[0]
439
440    def tc_add_ingress(self):
441        tc("qdisc add dev %s ingress" % (self['ifname']))
442
443    def tc_del_ingress(self):
444        tc("qdisc del dev %s ingress" % (self['ifname']))
445
446    def tc_flush_filters(self, bound=0, total=0):
447        self.tc_del_ingress()
448        self.tc_add_ingress()
449        self.wait_for_flush(bound=bound, total=total)
450
451    def tc_show_ingress(self, expected=None):
452        # No JSON support, oh well...
453        flags = ["skip_sw", "skip_hw", "in_hw"]
454        named = ["protocol", "pref", "chain", "handle", "id", "tag"]
455
456        args = "-s filter show dev %s ingress" % (self['ifname'])
457        _, out = tc(args, JSON=False)
458
459        filters = []
460        lines = out.split('\n')
461        for line in lines:
462            words = line.split()
463            if "handle" not in words:
464                continue
465            fltr = {}
466            for flag in flags:
467                fltr[flag] = flag in words
468            for name in named:
469                try:
470                    idx = words.index(name)
471                    fltr[name] = words[idx + 1]
472                except ValueError:
473                    pass
474            filters.append(fltr)
475
476        if expected is not None:
477            fail(len(filters) != expected,
478                 "%d ingress filters loaded, expected %d" %
479                 (len(filters), expected))
480        return filters
481
482    def cls_filter_op(self, op, qdisc="ingress", prio=None, handle=None,
483                      chain=None, cls="", params="",
484                      fail=True, include_stderr=False):
485        spec = ""
486        if prio is not None:
487            spec += " prio %d" % (prio)
488        if handle:
489            spec += " handle %s" % (handle)
490        if chain is not None:
491            spec += " chain %d" % (chain)
492
493        return tc("filter {op} dev {dev} {qdisc} {spec} {cls} {params}"\
494                  .format(op=op, dev=self['ifname'], qdisc=qdisc, spec=spec,
495                          cls=cls, params=params),
496                  fail=fail, include_stderr=include_stderr)
497
498    def cls_bpf_add_filter(self, bpf, op="add", prio=None, handle=None,
499                           chain=None, da=False, verbose=False,
500                           skip_sw=False, skip_hw=False,
501                           fail=True, include_stderr=False):
502        cls = "bpf " + bpf
503
504        params = ""
505        if da:
506            params += " da"
507        if verbose:
508            params += " verbose"
509        if skip_sw:
510            params += " skip_sw"
511        if skip_hw:
512            params += " skip_hw"
513
514        return self.cls_filter_op(op=op, prio=prio, handle=handle, cls=cls,
515                                  chain=chain, params=params,
516                                  fail=fail, include_stderr=include_stderr)
517
518    def set_ethtool_tc_offloads(self, enable, fail=True):
519        args = "hw-tc-offload %s" % ("on" if enable else "off")
520        return ethtool(self, "-K", args, fail=fail)
521
522################################################################################
523def clean_up():
524    global files, netns, devs
525
526    for dev in devs:
527        dev.remove()
528    for f in files:
529        cmd("rm -f %s" % (f))
530    for ns in netns:
531        cmd("ip netns delete %s" % (ns))
532    files = []
533    netns = []
534
535def pin_prog(file_name, idx=0):
536    progs = bpftool_prog_list(expected=(idx + 1))
537    prog = progs[idx]
538    bpftool("prog pin id %d %s" % (prog["id"], file_name))
539    files.append(file_name)
540
541    return file_name, bpf_pinned(file_name)
542
543def pin_map(file_name, idx=0, expected=1):
544    maps = bpftool_map_list_wait(expected=expected)
545    m = maps[idx]
546    bpftool("map pin id %d %s" % (m["id"], file_name))
547    files.append(file_name)
548
549    return file_name, bpf_pinned(file_name)
550
551def check_dev_info_removed(prog_file=None, map_file=None):
552    bpftool_prog_list(expected=0)
553    bpftool_prog_list(expected=1, exclude_orphaned=False)
554    ret, err = bpftool("prog show pin %s" % (prog_file), fail=False)
555    fail(ret != 0, "failed to show prog with removed device")
556
557    bpftool_map_list_wait(expected=0)
558    ret, err = bpftool("map show pin %s" % (map_file), fail=False)
559    fail(ret == 0, "Showing map with removed device did not fail")
560    fail(err["error"].find("No such device") == -1,
561         "Showing map with removed device expected ENODEV, error is %s" %
562         (err["error"]))
563
564def check_dev_info(other_ns, ns, prog_file=None, map_file=None, removed=False):
565    progs = bpftool_prog_list(expected=1, ns=ns)
566    prog = progs[0]
567
568    fail("dev" not in prog.keys(), "Device parameters not reported")
569    dev = prog["dev"]
570    fail("ifindex" not in dev.keys(), "Device parameters not reported")
571    fail("ns_dev" not in dev.keys(), "Device parameters not reported")
572    fail("ns_inode" not in dev.keys(), "Device parameters not reported")
573
574    if not other_ns:
575        fail("ifname" not in dev.keys(), "Ifname not reported")
576        fail(dev["ifname"] != sim["ifname"],
577             "Ifname incorrect %s vs %s" % (dev["ifname"], sim["ifname"]))
578    else:
579        fail("ifname" in dev.keys(), "Ifname is reported for other ns")
580
581    maps = bpftool_map_list_wait(expected=2, ns=ns)
582    for m in maps:
583        fail("dev" not in m.keys(), "Device parameters not reported")
584        fail(dev != m["dev"], "Map's device different than program's")
585
586def check_extack(output, reference, args):
587    if skip_extack:
588        return
589    lines = output.split("\n")
590    comp = len(lines) >= 2 and lines[1] == 'Error: ' + reference
591    fail(not comp, "Missing or incorrect netlink extack message")
592
593def check_extack_nsim(output, reference, args):
594    check_extack(output, "netdevsim: " + reference, args)
595
596def check_no_extack(res, needle):
597    fail((res[1] + res[2]).count(needle) or (res[1] + res[2]).count("Warning:"),
598         "Found '%s' in command output, leaky extack?" % (needle))
599
600def check_verifier_log(output, reference):
601    lines = output.split("\n")
602    for l in reversed(lines):
603        if l == reference:
604            return
605    fail(True, "Missing or incorrect message from netdevsim in verifier log")
606
607def check_multi_basic(two_xdps):
608    fail(two_xdps["mode"] != 4, "Bad mode reported with multiple programs")
609    fail("prog" in two_xdps, "Base program reported in multi program mode")
610    fail(len(two_xdps["attached"]) != 2,
611         "Wrong attached program count with two programs")
612    fail(two_xdps["attached"][0]["prog"]["id"] ==
613         two_xdps["attached"][1]["prog"]["id"],
614         "Offloaded and other programs have the same id")
615
616def test_spurios_extack(sim, obj, skip_hw, needle):
617    res = sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=skip_hw,
618                                 include_stderr=True)
619    check_no_extack(res, needle)
620    res = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
621                                 skip_hw=skip_hw, include_stderr=True)
622    check_no_extack(res, needle)
623    res = sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf",
624                            include_stderr=True)
625    check_no_extack(res, needle)
626
627def test_multi_prog(simdev, sim, obj, modename, modeid):
628    start_test("Test multi-attachment XDP - %s + offload..." %
629               (modename or "default", ))
630    sim.set_xdp(obj, "offload")
631    xdp = sim.ip_link_show(xdp=True)["xdp"]
632    offloaded = sim.dfs_read("bpf_offloaded_id")
633    fail("prog" not in xdp, "Base program not reported in single program mode")
634    fail(len(xdp["attached"]) != 1,
635         "Wrong attached program count with one program")
636
637    sim.set_xdp(obj, modename)
638    two_xdps = sim.ip_link_show(xdp=True)["xdp"]
639
640    fail(xdp["attached"][0] not in two_xdps["attached"],
641         "Offload program not reported after other activated")
642    check_multi_basic(two_xdps)
643
644    offloaded2 = sim.dfs_read("bpf_offloaded_id")
645    fail(offloaded != offloaded2,
646         "Offload ID changed after loading other program")
647
648    start_test("Test multi-attachment XDP - replace...")
649    ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
650    fail(ret == 0, "Replaced one of programs without -force")
651    check_extack(err, "XDP program already attached.", args)
652
653    start_test("Test multi-attachment XDP - remove without mode...")
654    ret, _, err = sim.unset_xdp("", force=True,
655                                fail=False, include_stderr=True)
656    fail(ret == 0, "Removed program without a mode flag")
657    check_extack(err, "More than one program loaded, unset mode is ambiguous.", args)
658
659    sim.unset_xdp("offload")
660    xdp = sim.ip_link_show(xdp=True)["xdp"]
661    offloaded = sim.dfs_read("bpf_offloaded_id")
662
663    fail(xdp["mode"] != modeid, "Bad mode reported after multiple programs")
664    fail("prog" not in xdp,
665         "Base program not reported after multi program mode")
666    fail(xdp["attached"][0] not in two_xdps["attached"],
667         "Offload program not reported after other activated")
668    fail(len(xdp["attached"]) != 1,
669         "Wrong attached program count with remaining programs")
670    fail(offloaded != "0", "Offload ID reported with only other program left")
671
672    start_test("Test multi-attachment XDP - reattach...")
673    sim.set_xdp(obj, "offload")
674    two_xdps = sim.ip_link_show(xdp=True)["xdp"]
675
676    fail(xdp["attached"][0] not in two_xdps["attached"],
677         "Other program not reported after offload activated")
678    check_multi_basic(two_xdps)
679
680    start_test("Test multi-attachment XDP - device remove...")
681    simdev.remove()
682
683    simdev = BpfNetdevSimDev()
684    sim, = simdev.nsims
685    sim.set_ethtool_tc_offloads(True)
686    return [simdev, sim]
687
688# Parse command line
689parser = argparse.ArgumentParser()
690parser.add_argument("--log", help="output verbose log to given file")
691args = parser.parse_args()
692if args.log:
693    logfile = open(args.log, 'w+')
694    logfile.write("# -*-Org-*-")
695
696log("Prepare...", "", level=1)
697log_level_inc()
698
699# Check permissions
700skip(os.getuid() != 0, "test must be run as root")
701
702# Check tools
703ret, progs = bpftool("prog", fail=False)
704skip(ret != 0, "bpftool not installed")
705base_progs = progs
706_, base_maps = bpftool("map")
707base_map_names = [
708    'pid_iter.rodata', # created on each bpftool invocation
709    'libbpf_det_bind', # created on each bpftool invocation
710]
711
712# Check netdevsim
713if not os.path.isdir("/sys/bus/netdevsim/"):
714    ret, out = cmd("modprobe netdevsim", fail=False)
715    skip(ret != 0, "netdevsim module could not be loaded")
716
717# Check debugfs
718_, out = cmd("mount")
719if out.find("/sys/kernel/debug type debugfs") == -1:
720    cmd("mount -t debugfs none /sys/kernel/debug")
721
722# Check samples are compiled
723samples = ["sample_ret0.bpf.o", "sample_map_ret0.bpf.o"]
724for s in samples:
725    ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)
726    skip(ret != 0, "sample %s/%s not found, please compile it" %
727         (bpf_test_dir, s))
728
729# Check if iproute2 is built with libmnl (needed by extack support)
730_, _, err = cmd("tc qdisc delete dev lo handle 0",
731                fail=False, include_stderr=True)
732if err.find("Error: Failed to find qdisc with specified handle.") == -1:
733    print("Warning: no extack message in iproute2 output, libmnl missing?")
734    log("Warning: no extack message in iproute2 output, libmnl missing?", "")
735    skip_extack = True
736
737# Check if net namespaces seem to work
738ns = mknetns()
739skip(ns is None, "Could not create a net namespace")
740cmd("ip netns delete %s" % (ns))
741netns = []
742
743try:
744    obj = bpf_obj("sample_ret0.bpf.o")
745    bytecode = bpf_bytecode("1,6 0 0 4294967295,")
746
747    start_test("Test destruction of generic XDP...")
748    simdev = BpfNetdevSimDev()
749    sim, = simdev.nsims
750    sim.set_xdp(obj, "generic")
751    simdev.remove()
752    bpftool_prog_list_wait(expected=0)
753
754    simdev = BpfNetdevSimDev()
755    sim, = simdev.nsims
756    sim.tc_add_ingress()
757
758    start_test("Test TC non-offloaded...")
759    ret, _ = sim.cls_bpf_add_filter(obj, skip_hw=True, fail=False)
760    fail(ret != 0, "Software TC filter did not load")
761
762    start_test("Test TC non-offloaded isn't getting bound...")
763    ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
764    fail(ret != 0, "Software TC filter did not load")
765    simdev.dfs_get_bound_progs(expected=0)
766
767    sim.tc_flush_filters()
768
769    start_test("Test TC offloads are off by default...")
770    ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
771                                         fail=False, include_stderr=True)
772    fail(ret == 0, "TC filter loaded without enabling TC offloads")
773    check_extack(err, "TC offload is disabled on net device.", args)
774    sim.wait_for_flush()
775
776    sim.set_ethtool_tc_offloads(True)
777    sim.dfs["bpf_tc_non_bound_accept"] = "Y"
778
779    start_test("Test TC offload by default...")
780    ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
781    fail(ret != 0, "Software TC filter did not load")
782    simdev.dfs_get_bound_progs(expected=0)
783    ingress = sim.tc_show_ingress(expected=1)
784    fltr = ingress[0]
785    fail(not fltr["in_hw"], "Filter not offloaded by default")
786
787    sim.tc_flush_filters()
788
789    start_test("Test TC cBPF bytcode tries offload by default...")
790    ret, _ = sim.cls_bpf_add_filter(bytecode, fail=False)
791    fail(ret != 0, "Software TC filter did not load")
792    simdev.dfs_get_bound_progs(expected=0)
793    ingress = sim.tc_show_ingress(expected=1)
794    fltr = ingress[0]
795    fail(not fltr["in_hw"], "Bytecode not offloaded by default")
796
797    sim.tc_flush_filters()
798    sim.dfs["bpf_tc_non_bound_accept"] = "N"
799
800    start_test("Test TC cBPF unbound bytecode doesn't offload...")
801    ret, _, err = sim.cls_bpf_add_filter(bytecode, skip_sw=True,
802                                         fail=False, include_stderr=True)
803    fail(ret == 0, "TC bytecode loaded for offload")
804    check_extack_nsim(err, "netdevsim configured to reject unbound programs.",
805                      args)
806    sim.wait_for_flush()
807
808    start_test("Test non-0 chain offload...")
809    ret, _, err = sim.cls_bpf_add_filter(obj, chain=1, prio=1, handle=1,
810                                         skip_sw=True,
811                                         fail=False, include_stderr=True)
812    fail(ret == 0, "Offloaded a filter to chain other than 0")
813    check_extack(err, "Driver supports only offload of chain 0.", args)
814    sim.tc_flush_filters()
815
816    start_test("Test TC replace...")
817    sim.cls_bpf_add_filter(obj, prio=1, handle=1)
818    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1)
819    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
820
821    sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_sw=True)
822    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_sw=True)
823    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
824
825    sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=True)
826    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_hw=True)
827    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
828
829    start_test("Test TC replace bad flags...")
830    for i in range(3):
831        for j in range(3):
832            ret, _ = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
833                                            skip_sw=(j == 1), skip_hw=(j == 2),
834                                            fail=False)
835            fail(bool(ret) != bool(j),
836                 "Software TC incorrect load in replace test, iteration %d" %
837                 (j))
838        sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
839
840    start_test("Test spurious extack from the driver...")
841    test_spurios_extack(sim, obj, False, "netdevsim")
842    test_spurios_extack(sim, obj, True, "netdevsim")
843
844    sim.set_ethtool_tc_offloads(False)
845
846    test_spurios_extack(sim, obj, False, "TC offload is disabled")
847    test_spurios_extack(sim, obj, True, "TC offload is disabled")
848
849    sim.set_ethtool_tc_offloads(True)
850
851    sim.tc_flush_filters()
852
853    start_test("Test TC offloads failure...")
854    sim.dfs["dev/bpf_bind_verifier_accept"] = 0
855    ret, _, err = sim.cls_bpf_add_filter(obj, verbose=True, skip_sw=True,
856                                         fail=False, include_stderr=True)
857    fail(ret == 0, "TC filter did not reject with TC offloads enabled")
858    check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
859    sim.dfs["dev/bpf_bind_verifier_accept"] = 1
860
861    start_test("Test TC offloads work...")
862    ret, _, err = sim.cls_bpf_add_filter(obj, verbose=True, skip_sw=True,
863                                         fail=False, include_stderr=True)
864    fail(ret != 0, "TC filter did not load with TC offloads enabled")
865
866    start_test("Test TC offload basics...")
867    dfs = simdev.dfs_get_bound_progs(expected=1)
868    progs = bpftool_prog_list(expected=1)
869    ingress = sim.tc_show_ingress(expected=1)
870
871    dprog = dfs[0]
872    prog = progs[0]
873    fltr = ingress[0]
874    fail(fltr["skip_hw"], "TC does reports 'skip_hw' on offloaded filter")
875    fail(not fltr["in_hw"], "TC does not report 'in_hw' for offloaded filter")
876    fail(not fltr["skip_sw"], "TC does not report 'skip_sw' back")
877
878    start_test("Test TC offload is device-bound...")
879    fail(str(prog["id"]) != fltr["id"], "Program IDs don't match")
880    fail(prog["tag"] != fltr["tag"], "Program tags don't match")
881    fail(fltr["id"] != dprog["id"], "Program IDs don't match")
882    fail(dprog["state"] != "xlated", "Offloaded program state not translated")
883    fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
884
885    start_test("Test disabling TC offloads is rejected while filters installed...")
886    ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
887    fail(ret == 0, "Driver should refuse to disable TC offloads with filters installed...")
888    sim.set_ethtool_tc_offloads(True)
889
890    start_test("Test qdisc removal frees things...")
891    sim.tc_flush_filters()
892    sim.tc_show_ingress(expected=0)
893
894    start_test("Test disabling TC offloads is OK without filters...")
895    ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
896    fail(ret != 0,
897         "Driver refused to disable TC offloads without filters installed...")
898
899    sim.set_ethtool_tc_offloads(True)
900
901    start_test("Test destroying device gets rid of TC filters...")
902    sim.cls_bpf_add_filter(obj, skip_sw=True)
903    simdev.remove()
904    bpftool_prog_list_wait(expected=0)
905
906    simdev = BpfNetdevSimDev()
907    sim, = simdev.nsims
908    sim.set_ethtool_tc_offloads(True)
909
910    start_test("Test destroying device gets rid of XDP...")
911    sim.set_xdp(obj, "offload")
912    simdev.remove()
913    bpftool_prog_list_wait(expected=0)
914
915    simdev = BpfNetdevSimDev()
916    sim, = simdev.nsims
917    sim.set_ethtool_tc_offloads(True)
918
919    start_test("Test XDP prog reporting...")
920    sim.set_xdp(obj, "drv")
921    ipl = sim.ip_link_show(xdp=True)
922    progs = bpftool_prog_list(expected=1)
923    fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
924         "Loaded program has wrong ID")
925
926    start_test("Test XDP prog replace without force...")
927    ret, _ = sim.set_xdp(obj, "drv", fail=False)
928    fail(ret == 0, "Replaced XDP program without -force")
929    sim.wait_for_flush(total=1)
930
931    start_test("Test XDP prog replace with force...")
932    ret, _ = sim.set_xdp(obj, "drv", force=True, fail=False)
933    fail(ret != 0, "Could not replace XDP program with -force")
934    bpftool_prog_list_wait(expected=1)
935    ipl = sim.ip_link_show(xdp=True)
936    progs = bpftool_prog_list(expected=1)
937    fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
938         "Loaded program has wrong ID")
939    fail("dev" in progs[0].keys(),
940         "Device parameters reported for non-offloaded program")
941
942    start_test("Test XDP prog replace with bad flags...")
943    ret, _, err = sim.set_xdp(obj, "generic", force=True,
944                              fail=False, include_stderr=True)
945    fail(ret == 0, "Replaced XDP program with a program in different mode")
946    check_extack(err,
947                 "Native and generic XDP can't be active at the same time.",
948                 args)
949
950    start_test("Test MTU restrictions...")
951    ret, _ = sim.set_mtu(9000, fail=False)
952    fail(ret == 0,
953         "Driver should refuse to increase MTU to 9000 with XDP loaded...")
954    sim.unset_xdp("drv")
955    bpftool_prog_list_wait(expected=0)
956    sim.set_mtu(9000)
957    ret, _, err = sim.set_xdp(obj, "drv", fail=False, include_stderr=True)
958    fail(ret == 0, "Driver should refuse to load program with MTU of 9000...")
959    check_extack_nsim(err, "MTU too large w/ XDP enabled.", args)
960    sim.set_mtu(1500)
961
962    sim.wait_for_flush()
963    start_test("Test non-offload XDP attaching to HW...")
964    bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/nooffload")
965    nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
966    ret, _, err = sim.set_xdp(nooffload, "offload",
967                              fail=False, include_stderr=True)
968    fail(ret == 0, "attached non-offloaded XDP program to HW")
969    check_extack_nsim(err, "xdpoffload of non-bound program.", args)
970    rm("/sys/fs/bpf/nooffload")
971
972    start_test("Test offload XDP attaching to drv...")
973    bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload",
974                      dev=sim['ifname'])
975    offload = bpf_pinned("/sys/fs/bpf/offload")
976    ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
977    fail(ret == 0, "attached offloaded XDP program to drv")
978    check_extack(err, "Using offloaded program without HW_MODE flag is not supported.", args)
979    rm("/sys/fs/bpf/offload")
980    sim.wait_for_flush()
981
982    start_test("Test XDP load failure...")
983    sim.dfs["dev/bpf_bind_verifier_accept"] = 0
984    ret, _, err = bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload",
985                                 dev=sim['ifname'], fail=False, include_stderr=True)
986    fail(ret == 0, "verifier should fail on load")
987    check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
988    sim.dfs["dev/bpf_bind_verifier_accept"] = 1
989    sim.wait_for_flush()
990
991    start_test("Test XDP offload...")
992    _, _, err = sim.set_xdp(obj, "offload", verbose=True, include_stderr=True)
993    ipl = sim.ip_link_show(xdp=True)
994    link_xdp = ipl["xdp"]["prog"]
995    progs = bpftool_prog_list(expected=1)
996    prog = progs[0]
997    fail(link_xdp["id"] != prog["id"], "Loaded program has wrong ID")
998
999    start_test("Test XDP offload is device bound...")
1000    dfs = simdev.dfs_get_bound_progs(expected=1)
1001    dprog = dfs[0]
1002
1003    fail(prog["id"] != link_xdp["id"], "Program IDs don't match")
1004    fail(prog["tag"] != link_xdp["tag"], "Program tags don't match")
1005    fail(str(link_xdp["id"]) != dprog["id"], "Program IDs don't match")
1006    fail(dprog["state"] != "xlated", "Offloaded program state not translated")
1007    fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
1008
1009    start_test("Test removing XDP program many times...")
1010    sim.unset_xdp("offload")
1011    sim.unset_xdp("offload")
1012    sim.unset_xdp("drv")
1013    sim.unset_xdp("drv")
1014    sim.unset_xdp("")
1015    sim.unset_xdp("")
1016    bpftool_prog_list_wait(expected=0)
1017
1018    start_test("Test attempt to use a program for a wrong device...")
1019    simdev2 = BpfNetdevSimDev()
1020    sim2, = simdev2.nsims
1021    sim2.set_xdp(obj, "offload")
1022    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
1023
1024    ret, _, err = sim.set_xdp(pinned, "offload",
1025                              fail=False, include_stderr=True)
1026    fail(ret == 0, "Pinned program loaded for a different device accepted")
1027    check_extack(err, "Program bound to different device.", args)
1028    simdev2.remove()
1029    ret, _, err = sim.set_xdp(pinned, "offload",
1030                              fail=False, include_stderr=True)
1031    fail(ret == 0, "Pinned program loaded for a removed device accepted")
1032    check_extack(err, "Program bound to different device.", args)
1033    rm(pin_file)
1034    bpftool_prog_list_wait(expected=0)
1035
1036    simdev, sim = test_multi_prog(simdev, sim, obj, "", 1)
1037    simdev, sim = test_multi_prog(simdev, sim, obj, "drv", 1)
1038    simdev, sim = test_multi_prog(simdev, sim, obj, "generic", 2)
1039
1040    start_test("Test mixing of TC and XDP...")
1041    sim.tc_add_ingress()
1042    sim.set_xdp(obj, "offload")
1043    ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
1044                                         fail=False, include_stderr=True)
1045    fail(ret == 0, "Loading TC when XDP active should fail")
1046    check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
1047    sim.unset_xdp("offload")
1048    sim.wait_for_flush()
1049
1050    sim.cls_bpf_add_filter(obj, skip_sw=True)
1051    ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
1052    fail(ret == 0, "Loading XDP when TC active should fail")
1053    check_extack_nsim(err, "TC program is already loaded.", args)
1054
1055    start_test("Test binding TC from pinned...")
1056    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
1057    sim.tc_flush_filters(bound=1, total=1)
1058    sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True)
1059    sim.tc_flush_filters(bound=1, total=1)
1060
1061    start_test("Test binding XDP from pinned...")
1062    sim.set_xdp(obj, "offload")
1063    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp2", idx=1)
1064
1065    sim.set_xdp(pinned, "offload", force=True)
1066    sim.unset_xdp("offload")
1067    sim.set_xdp(pinned, "offload", force=True)
1068    sim.unset_xdp("offload")
1069
1070    start_test("Test offload of wrong type fails...")
1071    ret, _ = sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True, fail=False)
1072    fail(ret == 0, "Managed to attach XDP program to TC")
1073
1074    start_test("Test asking for TC offload of two filters...")
1075    sim.cls_bpf_add_filter(obj, da=True, skip_sw=True)
1076    ret, _, err = sim.cls_bpf_add_filter(obj, da=True, skip_sw=True,
1077                                         fail=False, include_stderr=True)
1078    fail(ret == 0, "Managed to offload two TC filters at the same time")
1079    check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
1080
1081    sim.tc_flush_filters(bound=2, total=2)
1082
1083    start_test("Test if netdev removal waits for translation...")
1084    delay_msec = 500
1085    sim.dfs["dev/bpf_bind_verifier_delay"] = delay_msec
1086    start = time.time()
1087    cmd_line = "tc filter add dev %s ingress bpf %s da skip_sw" % \
1088               (sim['ifname'], obj)
1089    tc_proc = cmd(cmd_line, background=True, fail=False)
1090    # Wait for the verifier to start
1091    while simdev.dfs_num_bound_progs() <= 2:
1092        pass
1093    simdev.remove()
1094    end = time.time()
1095    ret, _ = cmd_result(tc_proc, fail=False)
1096    time_diff = end - start
1097    log("Time", "start:\t%s\nend:\t%s\ndiff:\t%s" % (start, end, time_diff))
1098
1099    fail(ret == 0, "Managed to load TC filter on a unregistering device")
1100    delay_sec = delay_msec * 0.001
1101    fail(time_diff < delay_sec, "Removal process took %s, expected %s" %
1102         (time_diff, delay_sec))
1103
1104    # Remove all pinned files and reinstantiate the netdev
1105    clean_up()
1106    bpftool_prog_list_wait(expected=0)
1107
1108    simdev = BpfNetdevSimDev()
1109    sim, = simdev.nsims
1110    map_obj = bpf_obj("sample_map_ret0.bpf.o")
1111    start_test("Test loading program with maps...")
1112    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1113
1114    start_test("Test bpftool bound info reporting (own ns)...")
1115    check_dev_info(False, "")
1116
1117    start_test("Test bpftool bound info reporting (other ns)...")
1118    ns = mknetns()
1119    sim.set_ns(ns)
1120    check_dev_info(True, "")
1121
1122    start_test("Test bpftool bound info reporting (remote ns)...")
1123    check_dev_info(False, ns)
1124
1125    start_test("Test bpftool bound info reporting (back to own ns)...")
1126    sim.set_ns("")
1127    check_dev_info(False, "")
1128
1129    prog_file, _ = pin_prog("/sys/fs/bpf/tmp_prog")
1130    map_file, _ = pin_map("/sys/fs/bpf/tmp_map", idx=1, expected=2)
1131    simdev.remove()
1132
1133    start_test("Test bpftool bound info reporting (removed dev)...")
1134    check_dev_info_removed(prog_file=prog_file, map_file=map_file)
1135
1136    # Remove all pinned files and reinstantiate the netdev
1137    clean_up()
1138    bpftool_prog_list_wait(expected=0)
1139
1140    simdev = BpfNetdevSimDev()
1141    sim, = simdev.nsims
1142
1143    start_test("Test map update (no flags)...")
1144    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1145    maps = bpftool_map_list_wait(expected=2)
1146    array = maps[0] if maps[0]["type"] == "array" else maps[1]
1147    htab = maps[0] if maps[0]["type"] == "hash" else maps[1]
1148    for m in maps:
1149        for i in range(2):
1150            bpftool("map update id %d key %s value %s" %
1151                    (m["id"], int2str("I", i), int2str("Q", i * 3)))
1152
1153    for m in maps:
1154        ret, _ = bpftool("map update id %d key %s value %s" %
1155                         (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1156                         fail=False)
1157        fail(ret == 0, "added too many entries")
1158
1159    start_test("Test map update (exists)...")
1160    for m in maps:
1161        for i in range(2):
1162            bpftool("map update id %d key %s value %s exist" %
1163                    (m["id"], int2str("I", i), int2str("Q", i * 3)))
1164
1165    for m in maps:
1166        ret, err = bpftool("map update id %d key %s value %s exist" %
1167                           (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1168                           fail=False)
1169        fail(ret == 0, "updated non-existing key")
1170        fail(err["error"].find("No such file or directory") == -1,
1171             "expected ENOENT, error is '%s'" % (err["error"]))
1172
1173    start_test("Test map update (noexist)...")
1174    for m in maps:
1175        for i in range(2):
1176            ret, err = bpftool("map update id %d key %s value %s noexist" %
1177                               (m["id"], int2str("I", i), int2str("Q", i * 3)),
1178                               fail=False)
1179        fail(ret == 0, "updated existing key")
1180        fail(err["error"].find("File exists") == -1,
1181             "expected EEXIST, error is '%s'" % (err["error"]))
1182
1183    start_test("Test map dump...")
1184    for m in maps:
1185        _, entries = bpftool("map dump id %d" % (m["id"]))
1186        for i in range(2):
1187            key = str2int(entries[i]["key"])
1188            fail(key != i, "expected key %d, got %d" % (key, i))
1189            val = str2int(entries[i]["value"])
1190            fail(val != i * 3, "expected value %d, got %d" % (val, i * 3))
1191
1192    start_test("Test map getnext...")
1193    for m in maps:
1194        _, entry = bpftool("map getnext id %d" % (m["id"]))
1195        key = str2int(entry["next_key"])
1196        fail(key != 0, "next key %d, expected %d" % (key, 0))
1197        _, entry = bpftool("map getnext id %d key %s" %
1198                           (m["id"], int2str("I", 0)))
1199        key = str2int(entry["next_key"])
1200        fail(key != 1, "next key %d, expected %d" % (key, 1))
1201        ret, err = bpftool("map getnext id %d key %s" %
1202                           (m["id"], int2str("I", 1)), fail=False)
1203        fail(ret == 0, "got next key past the end of map")
1204        fail(err["error"].find("No such file or directory") == -1,
1205             "expected ENOENT, error is '%s'" % (err["error"]))
1206
1207    start_test("Test map delete (htab)...")
1208    for i in range(2):
1209        bpftool("map delete id %d key %s" % (htab["id"], int2str("I", i)))
1210
1211    start_test("Test map delete (array)...")
1212    for i in range(2):
1213        ret, err = bpftool("map delete id %d key %s" %
1214                           (htab["id"], int2str("I", i)), fail=False)
1215        fail(ret == 0, "removed entry from an array")
1216        fail(err["error"].find("No such file or directory") == -1,
1217             "expected ENOENT, error is '%s'" % (err["error"]))
1218
1219    start_test("Test map remove...")
1220    sim.unset_xdp("offload")
1221    bpftool_map_list_wait(expected=0)
1222    simdev.remove()
1223
1224    simdev = BpfNetdevSimDev()
1225    sim, = simdev.nsims
1226    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1227    simdev.remove()
1228    bpftool_map_list_wait(expected=0)
1229
1230    start_test("Test map creation fail path...")
1231    simdev = BpfNetdevSimDev()
1232    sim, = simdev.nsims
1233    sim.dfs["bpf_map_accept"] = "N"
1234    ret, _ = sim.set_xdp(map_obj, "offload", JSON=False, fail=False)
1235    fail(ret == 0,
1236         "netdevsim didn't refuse to create a map with offload disabled")
1237
1238    simdev.remove()
1239
1240    start_test("Test multi-dev ASIC program reuse...")
1241    simdevA = BpfNetdevSimDev()
1242    simA, = simdevA.nsims
1243    simdevB = BpfNetdevSimDev(3)
1244    simB1, simB2, simB3 = simdevB.nsims
1245    sims = (simA, simB1, simB2, simB3)
1246    simB = (simB1, simB2, simB3)
1247
1248    bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA",
1249                      dev=simA['ifname'])
1250    progA = bpf_pinned("/sys/fs/bpf/nsimA")
1251    bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB",
1252                      dev=simB1['ifname'])
1253    progB = bpf_pinned("/sys/fs/bpf/nsimB")
1254
1255    simA.set_xdp(progA, "offload", JSON=False)
1256    for d in simdevB.nsims:
1257        d.set_xdp(progB, "offload", JSON=False)
1258
1259    start_test("Test multi-dev ASIC cross-dev replace...")
1260    ret, _ = simA.set_xdp(progB, "offload", force=True, JSON=False, fail=False)
1261    fail(ret == 0, "cross-ASIC program allowed")
1262    for d in simdevB.nsims:
1263        ret, _ = d.set_xdp(progA, "offload", force=True, JSON=False, fail=False)
1264        fail(ret == 0, "cross-ASIC program allowed")
1265
1266    start_test("Test multi-dev ASIC cross-dev install...")
1267    for d in sims:
1268        d.unset_xdp("offload")
1269
1270    ret, _, err = simA.set_xdp(progB, "offload", force=True, JSON=False,
1271                               fail=False, include_stderr=True)
1272    fail(ret == 0, "cross-ASIC program allowed")
1273    check_extack(err, "Program bound to different device.", args)
1274    for d in simdevB.nsims:
1275        ret, _, err = d.set_xdp(progA, "offload", force=True, JSON=False,
1276                                fail=False, include_stderr=True)
1277        fail(ret == 0, "cross-ASIC program allowed")
1278        check_extack(err, "Program bound to different device.", args)
1279
1280    start_test("Test multi-dev ASIC cross-dev map reuse...")
1281
1282    mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
1283    mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
1284
1285    ret, _ = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_",
1286                               dev=simB3['ifname'],
1287                               maps=["idx 0 id %d" % (mapB)],
1288                               fail=False)
1289    fail(ret != 0, "couldn't reuse a map on the same ASIC")
1290    rm("/sys/fs/bpf/nsimB_")
1291
1292    ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA_",
1293                                    dev=simA['ifname'],
1294                                    maps=["idx 0 id %d" % (mapB)],
1295                                    fail=False, include_stderr=True)
1296    fail(ret == 0, "could reuse a map on a different ASIC")
1297    fail(err.count("offload device mismatch between prog and map") == 0,
1298         "error message missing for cross-ASIC map")
1299
1300    ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_",
1301                                    dev=simB1['ifname'],
1302                                    maps=["idx 0 id %d" % (mapA)],
1303                                    fail=False, include_stderr=True)
1304    fail(ret == 0, "could reuse a map on a different ASIC")
1305    fail(err.count("offload device mismatch between prog and map") == 0,
1306         "error message missing for cross-ASIC map")
1307
1308    start_test("Test multi-dev ASIC cross-dev destruction...")
1309    bpftool_prog_list_wait(expected=2)
1310
1311    simdevA.remove()
1312    bpftool_prog_list_wait(expected=1)
1313
1314    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1315    fail(ifnameB != simB1['ifname'], "program not bound to original device")
1316    simB1.remove()
1317    bpftool_prog_list_wait(expected=1)
1318
1319    start_test("Test multi-dev ASIC cross-dev destruction - move...")
1320    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1321    fail(ifnameB not in (simB2['ifname'], simB3['ifname']),
1322         "program not bound to remaining devices")
1323
1324    simB2.remove()
1325    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1326    fail(ifnameB != simB3['ifname'], "program not bound to remaining device")
1327
1328    simB3.remove()
1329    simdevB.remove()
1330    bpftool_prog_list_wait(expected=0)
1331
1332    start_test("Test multi-dev ASIC cross-dev destruction - orphaned...")
1333    ret, out = bpftool("prog show %s" % (progB), fail=False)
1334    fail(ret != 0, "couldn't get information about orphaned program")
1335
1336    print("%s: OK" % (os.path.basename(__file__)))
1337
1338finally:
1339    log("Clean up...", "", level=1)
1340    log_level_inc()
1341    clean_up()
1342