1 #! /bin/bash 2 # SPDX-License-Identifier: GPL-2.0 3 4 . "$(dirname "${0}")/../lib.sh" 5 . "$(dirname "${0}")/../net_helper.sh" 6 7 readonly KSFT_PASS=0 8 readonly KSFT_FAIL=1 9 readonly KSFT_SKIP=4 10 11 # shellcheck disable=SC2155 # declare and assign separately 12 readonly KSFT_TEST="${MPTCP_LIB_KSFT_TEST:-$(basename "${0}" .sh)}" 13 14 # These variables are used in some selftests, read-only 15 declare -rx MPTCP_LIB_EVENT_CREATED=1 # MPTCP_EVENT_CREATED 16 declare -rx MPTCP_LIB_EVENT_ESTABLISHED=2 # MPTCP_EVENT_ESTABLISHED 17 declare -rx MPTCP_LIB_EVENT_CLOSED=3 # MPTCP_EVENT_CLOSED 18 declare -rx MPTCP_LIB_EVENT_ANNOUNCED=6 # MPTCP_EVENT_ANNOUNCED 19 declare -rx MPTCP_LIB_EVENT_REMOVED=7 # MPTCP_EVENT_REMOVED 20 declare -rx MPTCP_LIB_EVENT_SUB_ESTABLISHED=10 # MPTCP_EVENT_SUB_ESTABLISHED 21 declare -rx MPTCP_LIB_EVENT_SUB_CLOSED=11 # MPTCP_EVENT_SUB_CLOSED 22 declare -rx MPTCP_LIB_EVENT_SUB_PRIORITY=13 # MPTCP_EVENT_SUB_PRIORITY 23 declare -rx MPTCP_LIB_EVENT_LISTENER_CREATED=15 # MPTCP_EVENT_LISTENER_CREATED 24 declare -rx MPTCP_LIB_EVENT_LISTENER_CLOSED=16 # MPTCP_EVENT_LISTENER_CLOSED 25 26 declare -rx MPTCP_LIB_AF_INET=2 27 declare -rx MPTCP_LIB_AF_INET6=10 28 29 MPTCP_LIB_SUBTESTS=() 30 MPTCP_LIB_SUBTESTS_DUPLICATED=0 31 MPTCP_LIB_SUBTEST_FLAKY=0 32 MPTCP_LIB_SUBTESTS_LAST_TS_MS= 33 MPTCP_LIB_TEST_COUNTER=0 34 MPTCP_LIB_TEST_FORMAT="%02u %-50s" 35 MPTCP_LIB_IP_MPTCP=0 36 37 # only if supported (or forced) and not disabled, see no-color.org 38 if { [ -t 1 ] || [ "${SELFTESTS_MPTCP_LIB_COLOR_FORCE:-}" = "1" ]; } && 39 [ "${NO_COLOR:-}" != "1" ]; then 40 readonly MPTCP_LIB_COLOR_RED="\E[1;31m" 41 readonly MPTCP_LIB_COLOR_GREEN="\E[1;32m" 42 readonly MPTCP_LIB_COLOR_YELLOW="\E[1;33m" 43 readonly MPTCP_LIB_COLOR_BLUE="\E[1;34m" 44 readonly MPTCP_LIB_COLOR_RESET="\E[0m" 45 else 46 readonly MPTCP_LIB_COLOR_RED= 47 readonly MPTCP_LIB_COLOR_GREEN= 48 readonly MPTCP_LIB_COLOR_YELLOW= 49 readonly MPTCP_LIB_COLOR_BLUE= 50 readonly MPTCP_LIB_COLOR_RESET= 51 fi 52 53 # SELFTESTS_MPTCP_LIB_OVERRIDE_FLAKY env var can be set not to ignore errors 54 # from subtests marked as flaky 55 mptcp_lib_override_flaky() { 56 [ "${SELFTESTS_MPTCP_LIB_OVERRIDE_FLAKY:-}" = 1 ] 57 } 58 59 mptcp_lib_subtest_is_flaky() { 60 [ "${MPTCP_LIB_SUBTEST_FLAKY}" = 1 ] && ! mptcp_lib_override_flaky 61 } 62 63 # $1: color, $2: text 64 mptcp_lib_print_color() { 65 echo -e "${MPTCP_LIB_START_PRINT:-}${*}${MPTCP_LIB_COLOR_RESET}" 66 } 67 68 mptcp_lib_print_ok() { 69 mptcp_lib_print_color "${MPTCP_LIB_COLOR_GREEN}${*}" 70 } 71 72 mptcp_lib_print_warn() { 73 mptcp_lib_print_color "${MPTCP_LIB_COLOR_YELLOW}${*}" 74 } 75 76 mptcp_lib_print_info() { 77 mptcp_lib_print_color "${MPTCP_LIB_COLOR_BLUE}${*}" 78 } 79 80 mptcp_lib_print_err() { 81 mptcp_lib_print_color "${MPTCP_LIB_COLOR_RED}${*}" 82 } 83 84 # shellcheck disable=SC2120 # parameters are optional 85 mptcp_lib_pr_ok() { 86 mptcp_lib_print_ok "[ OK ]${1:+ ${*}}" 87 } 88 89 mptcp_lib_pr_skip() { 90 mptcp_lib_print_warn "[SKIP]${1:+ ${*}}" 91 } 92 93 mptcp_lib_pr_fail() { 94 local title cmt 95 96 if mptcp_lib_subtest_is_flaky; then 97 title="IGNO" 98 cmt=" (flaky)" 99 else 100 title="FAIL" 101 fi 102 103 mptcp_lib_print_err "[${title}]${cmt}${1:+ ${*}}" 104 } 105 106 mptcp_lib_pr_info() { 107 mptcp_lib_print_info "INFO: ${*}" 108 } 109 110 # SELFTESTS_MPTCP_LIB_EXPECT_ALL_FEATURES env var can be set when validating all 111 # features using the last version of the kernel and the selftests to make sure 112 # a test is not being skipped by mistake. 113 mptcp_lib_expect_all_features() { 114 [ "${SELFTESTS_MPTCP_LIB_EXPECT_ALL_FEATURES:-}" = "1" ] 115 } 116 117 # $1: msg 118 mptcp_lib_fail_if_expected_feature() { 119 if mptcp_lib_expect_all_features; then 120 echo "ERROR: missing feature: ${*}" 121 exit ${KSFT_FAIL} 122 fi 123 124 return 1 125 } 126 127 # $1: file 128 mptcp_lib_has_file() { 129 local f="${1}" 130 131 if [ -f "${f}" ]; then 132 return 0 133 fi 134 135 mptcp_lib_fail_if_expected_feature "${f} file not found" 136 } 137 138 mptcp_lib_check_mptcp() { 139 if ! mptcp_lib_has_file "/proc/sys/net/mptcp/enabled"; then 140 mptcp_lib_pr_skip "MPTCP support is not available" 141 exit ${KSFT_SKIP} 142 fi 143 } 144 145 mptcp_lib_check_kallsyms() { 146 if ! mptcp_lib_has_file "/proc/kallsyms"; then 147 mptcp_lib_pr_skip "CONFIG_KALLSYMS is missing" 148 exit ${KSFT_SKIP} 149 fi 150 } 151 152 # Internal: use mptcp_lib_kallsyms_has() instead 153 __mptcp_lib_kallsyms_has() { 154 local sym="${1}" 155 156 mptcp_lib_check_kallsyms 157 158 grep -q " ${sym}" /proc/kallsyms 159 } 160 161 # $1: part of a symbol to look at, add '$' at the end for full name 162 mptcp_lib_kallsyms_has() { 163 local sym="${1}" 164 165 if __mptcp_lib_kallsyms_has "${sym}"; then 166 return 0 167 fi 168 169 mptcp_lib_fail_if_expected_feature "${sym} symbol not found" 170 } 171 172 # $1: part of a symbol to look at, add '$' at the end for full name 173 mptcp_lib_kallsyms_doesnt_have() { 174 local sym="${1}" 175 176 if ! __mptcp_lib_kallsyms_has "${sym}"; then 177 return 0 178 fi 179 180 mptcp_lib_fail_if_expected_feature "${sym} symbol has been found" 181 } 182 183 # !!!AVOID USING THIS!!! 184 # Features might not land in the expected version and features can be backported 185 # 186 # $1: kernel version, e.g. 6.3 187 mptcp_lib_kversion_ge() { 188 local exp_maj="${1%.*}" 189 local exp_min="${1#*.}" 190 local v maj min 191 192 # If the kernel has backported features, set this env var to 1: 193 if [ "${SELFTESTS_MPTCP_LIB_NO_KVERSION_CHECK:-}" = "1" ]; then 194 return 0 195 fi 196 197 v=$(uname -r | cut -d'.' -f1,2) 198 maj=${v%.*} 199 min=${v#*.} 200 201 if [ "${maj}" -gt "${exp_maj}" ] || 202 { [ "${maj}" -eq "${exp_maj}" ] && [ "${min}" -ge "${exp_min}" ]; }; then 203 return 0 204 fi 205 206 mptcp_lib_fail_if_expected_feature "kernel version ${1} lower than ${v}" 207 } 208 209 mptcp_lib_subtests_last_ts_reset() { 210 MPTCP_LIB_SUBTESTS_LAST_TS_MS="$(date +%s%3N)" 211 } 212 mptcp_lib_subtests_last_ts_reset 213 214 __mptcp_lib_result_check_duplicated() { 215 local subtest 216 217 for subtest in "${MPTCP_LIB_SUBTESTS[@]}"; do 218 if [[ "${subtest}" == *" - ${KSFT_TEST}: ${*%% #*}" ]]; then 219 MPTCP_LIB_SUBTESTS_DUPLICATED=1 220 mptcp_lib_print_err "Duplicated entry: ${*}" 221 break 222 fi 223 done 224 } 225 226 __mptcp_lib_result_add() { 227 local result="${1}" 228 local time="time=" 229 local ts_prev_ms 230 shift 231 232 local id=$((${#MPTCP_LIB_SUBTESTS[@]} + 1)) 233 234 __mptcp_lib_result_check_duplicated "${*}" 235 236 # not to add two '#' 237 [[ "${*}" != *"#"* ]] && time="# ${time}" 238 239 ts_prev_ms="${MPTCP_LIB_SUBTESTS_LAST_TS_MS}" 240 mptcp_lib_subtests_last_ts_reset 241 time+="$((MPTCP_LIB_SUBTESTS_LAST_TS_MS - ts_prev_ms))ms" 242 243 MPTCP_LIB_SUBTESTS+=("${result} ${id} - ${KSFT_TEST}: ${*} ${time}") 244 } 245 246 # $1: test name 247 mptcp_lib_result_pass() { 248 __mptcp_lib_result_add "ok" "${1}" 249 } 250 251 # $1: test name 252 mptcp_lib_result_fail() { 253 if mptcp_lib_subtest_is_flaky; then 254 # It might sound better to use 'not ok # TODO' or 'ok # SKIP', 255 # but some CIs don't understand 'TODO' and treat SKIP as errors. 256 __mptcp_lib_result_add "ok" "${1} # IGNORE Flaky" 257 else 258 __mptcp_lib_result_add "not ok" "${1}" 259 fi 260 } 261 262 # $1: test name 263 mptcp_lib_result_skip() { 264 __mptcp_lib_result_add "ok" "${1} # SKIP" 265 } 266 267 # $1: result code ; $2: test name 268 mptcp_lib_result_code() { 269 local ret="${1}" 270 local name="${2}" 271 272 case "${ret}" in 273 "${KSFT_PASS}") 274 mptcp_lib_result_pass "${name}" 275 ;; 276 "${KSFT_FAIL}") 277 mptcp_lib_result_fail "${name}" 278 ;; 279 "${KSFT_SKIP}") 280 mptcp_lib_result_skip "${name}" 281 ;; 282 *) 283 echo "ERROR: wrong result code: ${ret}" 284 exit ${KSFT_FAIL} 285 ;; 286 esac 287 } 288 289 mptcp_lib_result_print_all_tap() { 290 local subtest 291 292 if [ ${#MPTCP_LIB_SUBTESTS[@]} -eq 0 ] || 293 [ "${SELFTESTS_MPTCP_LIB_NO_TAP:-}" = "1" ]; then 294 return 295 fi 296 297 printf "\nTAP version 13\n" 298 printf "1..%d\n" "${#MPTCP_LIB_SUBTESTS[@]}" 299 300 for subtest in "${MPTCP_LIB_SUBTESTS[@]}"; do 301 printf "%s\n" "${subtest}" 302 done 303 304 if [ "${MPTCP_LIB_SUBTESTS_DUPLICATED}" = 1 ] && 305 mptcp_lib_expect_all_features; then 306 mptcp_lib_print_err "Duplicated test entries" 307 exit ${KSFT_FAIL} 308 fi 309 } 310 311 # get the value of keyword $1 in the line marked by keyword $2 312 mptcp_lib_get_info_value() { 313 grep "${2}" | sed -n 's/.*\('"${1}"':\)\([0-9a-f:.]*\).*$/\2/p;q' 314 } 315 316 # $1: info name ; $2: evts_ns ; [$3: event type; [$4: addr]] 317 mptcp_lib_evts_get_info() { 318 grep "${4:-}" "${2}" | mptcp_lib_get_info_value "${1}" "^type:${3:-1}," 319 } 320 321 # $1: PID 322 mptcp_lib_kill_wait() { 323 [ "${1}" -eq 0 ] && return 0 324 325 kill -SIGUSR1 "${1}" > /dev/null 2>&1 326 kill "${1}" > /dev/null 2>&1 327 wait "${1}" 2>/dev/null 328 } 329 330 # $1: IP address 331 mptcp_lib_is_v6() { 332 [ -z "${1##*:*}" ] 333 } 334 335 # $1: ns, $2: MIB counter 336 mptcp_lib_get_counter() { 337 local ns="${1}" 338 local counter="${2}" 339 local count 340 341 count=$(ip netns exec "${ns}" nstat -asz "${counter}" | 342 awk 'NR==1 {next} {print $2}') 343 if [ -z "${count}" ]; then 344 mptcp_lib_fail_if_expected_feature "${counter} counter" 345 return 1 346 fi 347 348 echo "${count}" 349 } 350 351 mptcp_lib_make_file() { 352 local name="${1}" 353 local bs="${2}" 354 local size="${3}" 355 356 dd if=/dev/urandom of="${name}" bs="${bs}" count="${size}" 2> /dev/null 357 echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "${name}" 358 } 359 360 # $1: file 361 mptcp_lib_print_file_err() { 362 ls -l "${1}" 1>&2 363 echo "Trailing bytes are: " 364 tail -c 27 "${1}" 365 } 366 367 # $1: input file ; $2: output file ; $3: what kind of file 368 mptcp_lib_check_transfer() { 369 local in="${1}" 370 local out="${2}" 371 local what="${3}" 372 373 if ! cmp "$in" "$out" > /dev/null 2>&1; then 374 mptcp_lib_pr_fail "$what does not match (in, out):" 375 mptcp_lib_print_file_err "$in" 376 mptcp_lib_print_file_err "$out" 377 378 return 1 379 fi 380 381 return 0 382 } 383 384 # $1: ns, $2: port 385 mptcp_lib_wait_local_port_listen() { 386 wait_local_port_listen "${@}" "tcp" 387 } 388 389 mptcp_lib_check_output() { 390 local err="${1}" 391 local cmd="${2}" 392 local expected="${3}" 393 local cmd_ret=0 394 local out 395 396 if ! out=$(${cmd} 2>"${err}"); then 397 cmd_ret=${?} 398 fi 399 400 if [ ${cmd_ret} -ne 0 ]; then 401 mptcp_lib_pr_fail "command execution '${cmd}' stderr" 402 cat "${err}" 403 return 2 404 elif [ "${out}" = "${expected}" ]; then 405 return 0 406 else 407 mptcp_lib_pr_fail "expected '${expected}' got '${out}'" 408 return 1 409 fi 410 } 411 412 mptcp_lib_check_tools() { 413 local tool 414 415 for tool in "${@}"; do 416 case "${tool}" in 417 "ip") 418 if ! ip -Version &> /dev/null; then 419 mptcp_lib_pr_skip "Could not run test without ip tool" 420 exit ${KSFT_SKIP} 421 fi 422 ;; 423 "tc") 424 if ! tc -help &> /dev/null; then 425 mptcp_lib_pr_skip "Could not run test without tc tool" 426 exit ${KSFT_SKIP} 427 fi 428 ;; 429 "ss") 430 if ! ss -h | grep -q MPTCP; then 431 mptcp_lib_pr_skip "ss tool does not support MPTCP" 432 exit ${KSFT_SKIP} 433 fi 434 ;; 435 "iptables"* | "ip6tables"*) 436 if ! "${tool}" -V &> /dev/null; then 437 mptcp_lib_pr_skip "Could not run all tests without ${tool}" 438 exit ${KSFT_SKIP} 439 fi 440 ;; 441 *) 442 mptcp_lib_pr_fail "Internal error: unsupported tool: ${tool}" 443 exit ${KSFT_FAIL} 444 ;; 445 esac 446 done 447 } 448 449 mptcp_lib_ns_init() { 450 if ! setup_ns "${@}"; then 451 mptcp_lib_pr_fail "Failed to setup namespaces ${*}" 452 exit ${KSFT_FAIL} 453 fi 454 455 local netns 456 for netns in "${@}"; do 457 ip netns exec "${!netns}" sysctl -q net.mptcp.enabled=1 458 ip netns exec "${!netns}" sysctl -q net.ipv4.conf.all.rp_filter=0 459 ip netns exec "${!netns}" sysctl -q net.ipv4.conf.default.rp_filter=0 460 done 461 } 462 463 mptcp_lib_ns_exit() { 464 cleanup_ns "${@}" 465 466 local netns 467 for netns in "${@}"; do 468 rm -f /tmp/"${netns}".{nstat,out} 469 done 470 } 471 472 mptcp_lib_events() { 473 local ns="${1}" 474 local evts="${2}" 475 declare -n pid="${3}" 476 477 :>"${evts}" 478 479 mptcp_lib_kill_wait "${pid:-0}" 480 ip netns exec "${ns}" ./pm_nl_ctl events >> "${evts}" 2>&1 & 481 pid=$! 482 } 483 484 mptcp_lib_print_title() { 485 : "${MPTCP_LIB_TEST_COUNTER:?}" 486 : "${MPTCP_LIB_TEST_FORMAT:?}" 487 488 # shellcheck disable=SC2059 # the format is in a variable 489 printf "${MPTCP_LIB_TEST_FORMAT}" "$((++MPTCP_LIB_TEST_COUNTER))" "${*}" 490 } 491 492 # $1: var name ; $2: prev ret 493 mptcp_lib_check_expected_one() { 494 local var="${1}" 495 local exp="e_${var}" 496 local prev_ret="${2}" 497 498 if [ "${!var}" = "${!exp}" ]; then 499 return 0 500 fi 501 502 if [ "${prev_ret}" = "0" ]; then 503 mptcp_lib_pr_fail 504 fi 505 506 mptcp_lib_print_err "Expected value for '${var}': '${!exp}', got '${!var}'." 507 return 1 508 } 509 510 # $@: all var names to check 511 mptcp_lib_check_expected() { 512 local rc=0 513 local var 514 515 for var in "${@}"; do 516 mptcp_lib_check_expected_one "${var}" "${rc}" || rc=1 517 done 518 519 return "${rc}" 520 } 521 522 # shellcheck disable=SC2034 # Some variables are used below but indirectly 523 mptcp_lib_verify_listener_events() { 524 local evt=${1} 525 local e_type=${2} 526 local e_family=${3} 527 local e_saddr=${4} 528 local e_sport=${5} 529 local type 530 local family 531 local saddr 532 local sport 533 local rc=0 534 535 type=$(mptcp_lib_evts_get_info type "${evt}" "${e_type}") 536 family=$(mptcp_lib_evts_get_info family "${evt}" "${e_type}") 537 if [ "${family}" ] && [ "${family}" = "${AF_INET6}" ]; then 538 saddr=$(mptcp_lib_evts_get_info saddr6 "${evt}" "${e_type}") 539 else 540 saddr=$(mptcp_lib_evts_get_info saddr4 "${evt}" "${e_type}") 541 fi 542 sport=$(mptcp_lib_evts_get_info sport "${evt}" "${e_type}") 543 544 mptcp_lib_check_expected "type" "family" "saddr" "sport" || rc="${?}" 545 return "${rc}" 546 } 547 548 mptcp_lib_set_ip_mptcp() { 549 MPTCP_LIB_IP_MPTCP=1 550 } 551 552 mptcp_lib_is_ip_mptcp() { 553 [ "${MPTCP_LIB_IP_MPTCP}" = "1" ] 554 } 555 556 # format: <id>,<ip>,<flags>,<dev> 557 mptcp_lib_pm_nl_format_endpoints() { 558 local entry id ip flags dev port 559 560 for entry in "${@}"; do 561 IFS=, read -r id ip flags dev port <<< "${entry}" 562 if mptcp_lib_is_ip_mptcp; then 563 echo -n "${ip}" 564 [ -n "${port}" ] && echo -n " port ${port}" 565 echo -n " id ${id}" 566 [ -n "${flags}" ] && echo -n " ${flags}" 567 [ -n "${dev}" ] && echo -n " dev ${dev}" 568 echo " " # always a space at the end 569 else 570 echo -n "id ${id}" 571 echo -n " flags ${flags//" "/","}" 572 [ -n "${dev}" ] && echo -n " dev ${dev}" 573 echo -n " ${ip}" 574 [ -n "${port}" ] && echo -n " ${port}" 575 echo "" 576 fi 577 done 578 } 579 580 mptcp_lib_pm_nl_get_endpoint() { 581 local ns=${1} 582 local id=${2} 583 584 if mptcp_lib_is_ip_mptcp; then 585 ip -n "${ns}" mptcp endpoint show id "${id}" 586 else 587 ip netns exec "${ns}" ./pm_nl_ctl get "${id}" 588 fi 589 } 590 591 mptcp_lib_pm_nl_set_limits() { 592 local ns=${1} 593 local addrs=${2} 594 local subflows=${3} 595 596 if mptcp_lib_is_ip_mptcp; then 597 ip -n "${ns}" mptcp limits set add_addr_accepted "${addrs}" subflows "${subflows}" 598 else 599 ip netns exec "${ns}" ./pm_nl_ctl limits "${addrs}" "${subflows}" 600 fi 601 } 602 603 mptcp_lib_pm_nl_add_endpoint() { 604 local ns=${1} 605 local addr=${2} 606 local flags dev id port 607 local nr=2 608 609 local p 610 for p in "${@}"; do 611 case "${p}" in 612 "flags" | "dev" | "id" | "port") 613 eval "${p}"=\$"${nr}" 614 ;; 615 esac 616 617 nr=$((nr + 1)) 618 done 619 620 if mptcp_lib_is_ip_mptcp; then 621 # shellcheck disable=SC2086 # blanks in flags, no double quote 622 ip -n "${ns}" mptcp endpoint add "${addr}" ${flags//","/" "} \ 623 ${dev:+dev "${dev}"} ${id:+id "${id}"} ${port:+port "${port}"} 624 else 625 ip netns exec "${ns}" ./pm_nl_ctl add "${addr}" ${flags:+flags "${flags}"} \ 626 ${dev:+dev "${dev}"} ${id:+id "${id}"} ${port:+port "${port}"} 627 fi 628 } 629 630 mptcp_lib_pm_nl_del_endpoint() { 631 local ns=${1} 632 local id=${2} 633 local addr=${3} 634 635 if mptcp_lib_is_ip_mptcp; then 636 [ "${id}" -ne 0 ] && addr='' 637 ip -n "${ns}" mptcp endpoint delete id "${id}" ${addr:+"${addr}"} 638 else 639 ip netns exec "${ns}" ./pm_nl_ctl del "${id}" "${addr}" 640 fi 641 } 642 643 mptcp_lib_pm_nl_flush_endpoint() { 644 local ns=${1} 645 646 if mptcp_lib_is_ip_mptcp; then 647 ip -n "${ns}" mptcp endpoint flush 648 else 649 ip netns exec "${ns}" ./pm_nl_ctl flush 650 fi 651 } 652 653 mptcp_lib_pm_nl_show_endpoints() { 654 local ns=${1} 655 656 if mptcp_lib_is_ip_mptcp; then 657 ip -n "${ns}" mptcp endpoint show 658 else 659 ip netns exec "${ns}" ./pm_nl_ctl dump 660 fi 661 } 662 663 mptcp_lib_pm_nl_change_endpoint() { 664 local ns=${1} 665 local id=${2} 666 local flags=${3} 667 668 if mptcp_lib_is_ip_mptcp; then 669 # shellcheck disable=SC2086 # blanks in flags, no double quote 670 ip -n "${ns}" mptcp endpoint change id "${id}" ${flags//","/" "} 671 else 672 ip netns exec "${ns}" ./pm_nl_ctl set id "${id}" flags "${flags}" 673 fi 674 } 675