1#!/usr/bin/env python3
2#
3# Test case executor
4# Copyright (c) 2013-2019, Jouni Malinen <j@w1.fi>
5#
6# This software may be distributed under the terms of the BSD license.
7# See README for more details.
8
9import os
10import re
11import gc
12import sys
13import time
14from datetime import datetime
15import argparse
16import subprocess
17import termios
18
19import logging
20logger = logging.getLogger()
21
22try:
23    import sqlite3
24    sqlite3_imported = True
25except ImportError:
26    sqlite3_imported = False
27
28scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
29sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy'))
30
31from wpasupplicant import WpaSupplicant
32from hostapd import HostapdGlobal
33from check_kernel import check_kernel
34from wlantest import Wlantest
35from utils import HwsimSkip
36
37def set_term_echo(fd, enabled):
38    [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] = termios.tcgetattr(fd)
39    if enabled:
40        lflag |= termios.ECHO
41    else:
42        lflag &= ~termios.ECHO
43    termios.tcsetattr(fd, termios.TCSANOW,
44                      [iflag, oflag, cflag, lflag, ispeed, ospeed, cc])
45
46def reset_devs(dev, apdev):
47    ok = True
48    for d in dev:
49        try:
50            d.reset()
51        except Exception as e:
52            logger.info("Failed to reset device " + d.ifname)
53            print(str(e))
54            ok = False
55
56    wpas = None
57    try:
58        wpas = WpaSupplicant(global_iface='/tmp/wpas-wlan5', monitor=False)
59        ifaces = wpas.global_request("INTERFACES").splitlines()
60        for iface in ifaces:
61            if iface.startswith("wlan"):
62                wpas.interface_remove(iface)
63    except Exception as e:
64        pass
65    if wpas:
66        wpas.close_ctrl()
67        del wpas
68
69    try:
70        hapd = HostapdGlobal()
71        hapd.flush()
72        ifaces = hapd.request("INTERFACES").splitlines()
73        for iface in ifaces:
74            if iface.startswith("wlan"):
75                hapd.remove(iface)
76        hapd.remove('as-erp')
77    except Exception as e:
78        logger.info("Failed to remove hostapd interface")
79        print(str(e))
80        ok = False
81    return ok
82
83def add_log_file(conn, test, run, type, path):
84    if not os.path.exists(path):
85        return
86    contents = None
87    with open(path, 'rb') as f:
88        contents = f.read()
89    if contents is None:
90        return
91    sql = "INSERT INTO logs(test,run,type,contents) VALUES(?, ?, ?, ?)"
92    params = (test, run, type, sqlite3.Binary(contents))
93    try:
94        conn.execute(sql, params)
95        conn.commit()
96    except Exception as e:
97        print("sqlite: " + str(e))
98        print("sql: %r" % (params, ))
99
100def report(conn, prefill, build, commit, run, test, result, duration, logdir,
101           sql_commit=True):
102    if conn:
103        if not build:
104            build = ''
105        if not commit:
106            commit = ''
107        if prefill:
108            conn.execute('DELETE FROM results WHERE test=? AND run=? AND result=?', (test, run, 'NOTRUN'))
109        sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES(?, ?, ?, ?, ?, ?, ?)"
110        params = (test, result, run, time.time(), duration, build, commit)
111        try:
112            conn.execute(sql, params)
113            if sql_commit:
114                conn.commit()
115        except Exception as e:
116            print("sqlite: " + str(e))
117            print("sql: %r" % (params, ))
118
119        if result == "FAIL":
120            for log in ["log", "log0", "log1", "log2", "log3", "log5",
121                        "hostapd", "dmesg", "hwsim0", "hwsim0.pcapng"]:
122                add_log_file(conn, test, run, log,
123                             logdir + "/" + test + "." + log)
124
125class DataCollector(object):
126    def __init__(self, logdir, testname, kmemleak, args):
127        self._logdir = logdir
128        self._testname = testname
129        self._tracing = args.tracing
130        self._dmesg = args.dmesg
131        self._kmemleak = kmemleak
132        self._dbus = args.dbus
133    def __enter__(self):
134        if self._tracing:
135            output = os.path.abspath(os.path.join(self._logdir, '%s.dat' % (self._testname, )))
136            self._trace_cmd = subprocess.Popen(['trace-cmd', 'record', '-o', output, '-T', '-e', 'skb', '-e', 'mac80211', '-e', 'cfg80211', '-e', 'printk', 'sh', '-c', 'echo STARTED ; read l'],
137                                               stdin=subprocess.PIPE,
138                                               stdout=subprocess.PIPE,
139                                               stderr=open('/dev/null', 'w'),
140                                               cwd=self._logdir)
141            l = self._trace_cmd.stdout.read(7)
142            while self._trace_cmd.poll() is None and b'STARTED' not in l:
143                l += self._trace_cmd.stdout.read(1)
144            res = self._trace_cmd.returncode
145            if res:
146                print("Failed calling trace-cmd: returned exit status %d" % res)
147                sys.exit(1)
148        if self._dbus:
149            output = os.path.abspath(os.path.join(self._logdir, '%s.dbus' % (self._testname, )))
150            self._dbus_cmd = subprocess.Popen(['dbus-monitor', '--system'],
151                                              stdout=open(output, 'w'),
152                                              stderr=open('/dev/null', 'w'),
153                                              cwd=self._logdir)
154            res = self._dbus_cmd.returncode
155            if res:
156                print("Failed calling dbus-monitor: returned exit status %d" % res)
157                sys.exit(1)
158    def __exit__(self, type, value, traceback):
159        if self._tracing:
160            self._trace_cmd.stdin.write(b'DONE\n')
161            self._trace_cmd.stdin.flush()
162            self._trace_cmd.wait()
163
164        if self._kmemleak:
165            output = os.path.join(self._logdir, '%s.kmemleak' % (self._testname, ))
166            num = 0
167            while os.path.exists(output):
168                output = os.path.join(self._logdir, '%s.kmemleak-%d' % (self._testname, num))
169                num += 1
170
171            # Trigger kmemleak
172            with open('/sys/kernel/debug/kmemleak', 'w+') as kmemleak:
173                kmemleak.write('scan')
174                kmemleak.seek(0)
175
176                # Minimum reporting age
177                time.sleep(5)
178
179                kmemleak.write('scan')
180                kmemleak.seek(0)
181
182                leaks = []
183                while l := kmemleak.read():
184                    leaks.append(l)
185                leaks = ''.join(leaks)
186                if leaks:
187                    with open(output, 'w') as out:
188                        out.write(leaks)
189
190                kmemleak.seek(0)
191                kmemleak.write('clear')
192
193        if self._dmesg:
194            output = os.path.join(self._logdir, '%s.dmesg' % (self._testname, ))
195            num = 0
196            while os.path.exists(output):
197                output = os.path.join(self._logdir, '%s.dmesg-%d' % (self._testname, num))
198                num += 1
199            subprocess.call(['dmesg', '-c'], stdout=open(output, 'w'))
200
201def rename_log(logdir, basename, testname, dev):
202    try:
203        import getpass
204        srcname = os.path.join(logdir, basename)
205        dstname = os.path.join(logdir, testname + '.' + basename)
206        num = 0
207        while os.path.exists(dstname):
208            dstname = os.path.join(logdir,
209                                   testname + '.' + basename + '-' + str(num))
210            num = num + 1
211        os.rename(srcname, dstname)
212        if dev:
213            dev.relog()
214            subprocess.call(['chown', '-f', getpass.getuser(), srcname])
215    except Exception as e:
216        logger.info("Failed to rename log files")
217        logger.info(e)
218
219def is_long_duration_test(t):
220    return hasattr(t, "long_duration_test") and t.long_duration_test
221
222def get_test_description(t):
223    if t.__doc__ is None:
224        desc = "MISSING DESCRIPTION"
225    else:
226        desc = t.__doc__
227    if is_long_duration_test(t):
228        desc += " [long]"
229    return desc
230
231def import_test_cases():
232    tests = []
233    test_modules = []
234    names = set()
235    files = os.listdir(scriptsdir)
236    re_files = (re.match(r'(test_.*)\.py$', n) for n in files)
237    test_files = (m.group(1) for m in re_files if m)
238    for t in test_files:
239        mod = __import__(t)
240        test_modules.append(mod.__name__.replace('test_', '', 1))
241        for key, val in mod.__dict__.items():
242            if key.startswith("test_"):
243                if val.__doc__ is None:
244                    print(f"Test case {val.__name__} misses __doc__")
245                tests.append(val)
246
247                name = val.__name__.replace('test_', '', 1)
248                if name in names:
249                    print(f"Test case {name} defined multiple times")
250                names.add(name)
251    return tests, test_modules, names
252
253def main():
254    tests, test_modules, test_names = import_test_cases()
255
256    run = None
257
258    parser = argparse.ArgumentParser(description='hwsim test runner')
259    parser.add_argument('--logdir', metavar='<directory>',
260                        help='log output directory for all other options, ' +
261                             'must be given if other log options are used')
262    group = parser.add_mutually_exclusive_group()
263    group.add_argument('-d', const=logging.DEBUG, action='store_const',
264                       dest='loglevel', default=logging.INFO,
265                       help="verbose debug output")
266    group.add_argument('-q', const=logging.WARNING, action='store_const',
267                       dest='loglevel', help="be quiet")
268
269    parser.add_argument('-S', metavar='<sqlite3 db>', dest='database',
270                        help='database to write results to')
271    parser.add_argument('--prefill-tests', action='store_true', dest='prefill',
272                        help='prefill test database with NOTRUN before all tests')
273    parser.add_argument('--commit', metavar='<commit id>',
274                        help='commit ID, only for database')
275    parser.add_argument('-b', metavar='<build>', dest='build', help='build ID')
276    parser.add_argument('-L', action='store_true', dest='update_tests_db',
277                        help='List tests (and update descriptions in DB)')
278    parser.add_argument('-T', action='store_true', dest='tracing',
279                        help='collect tracing per test case (in log directory)')
280    parser.add_argument('-D', action='store_true', dest='dmesg',
281                        help='collect dmesg per test case (in log directory)')
282    parser.add_argument('--dbus', action='store_true', dest='dbus',
283                        help='collect dbus per test case (in log directory)')
284    parser.add_argument('--shuffle-tests', action='store_true',
285                        dest='shuffle_tests',
286                        help='Shuffle test cases to randomize order')
287    parser.add_argument('--split', help='split tests for parallel execution (<server number>/<total servers>)')
288    parser.add_argument('--no-reset', action='store_true', dest='no_reset',
289                        help='Do not reset devices at the end of the test')
290    parser.add_argument('--long', action='store_true',
291                        help='Include test cases that take long time')
292    parser.add_argument('-f', dest='testmodules', metavar='<test module>',
293                        help='execute only tests from these test modules',
294                        type=str, choices=[[]] + test_modules, nargs='+')
295    parser.add_argument('-l', metavar='<modules file>', dest='mfile',
296                        help='test modules file name')
297    parser.add_argument('-i', action='store_true', dest='stdin_ctrl',
298                        help='stdin-controlled test case execution')
299    parser.add_argument('tests', metavar='<test>', nargs='*', type=str,
300                        help='tests to run (only valid without -f)')
301
302    args = parser.parse_args()
303
304    if (args.tests and args.testmodules) or (args.tests and args.mfile) or (args.testmodules and args.mfile):
305        print('Invalid arguments - only one of (test, test modules, modules file) can be given.')
306        sys.exit(2)
307
308    if args.tests:
309        fail = False
310        for t in args.tests:
311            if t.endswith('*'):
312                prefix = t.rstrip('*')
313                found = False
314                for tn in test_names:
315                    if tn.startswith(prefix):
316                        found = True
317                        break
318                if not found:
319                    print('Invalid arguments - test "%s" wildcard did not match' % t)
320                    fail = True
321            elif t not in test_names:
322                print('Invalid arguments - test "%s" not known' % t)
323                fail = True
324        if fail:
325            sys.exit(2)
326
327    if args.database:
328        if not sqlite3_imported:
329            print("No sqlite3 module found")
330            sys.exit(2)
331        conn = sqlite3.connect(args.database)
332        conn.execute('CREATE TABLE IF NOT EXISTS results (test,result,run,time,duration,build,commitid)')
333        conn.execute('CREATE TABLE IF NOT EXISTS tests (test,description)')
334        conn.execute('CREATE TABLE IF NOT EXISTS logs (test,run,type,contents)')
335    else:
336        conn = None
337
338    if conn:
339        run = int(time.time())
340
341    # read the modules from the modules file
342    if args.mfile:
343        args.testmodules = []
344        with open(args.mfile) as f:
345            for line in f.readlines():
346                line = line.strip()
347                if not line or line.startswith('#'):
348                    continue
349                args.testmodules.append(line)
350
351    tests_to_run = []
352    if args.tests:
353        for selected in args.tests:
354            for t in tests:
355                name = t.__name__.replace('test_', '', 1)
356                if selected.endswith('*'):
357                    prefix = selected.rstrip('*')
358                    if name.startswith(prefix):
359                        tests_to_run.append(t)
360                elif name == selected:
361                    tests_to_run.append(t)
362    else:
363        for t in tests:
364            name = t.__name__.replace('test_', '', 1)
365            if args.testmodules:
366                if t.__module__.replace('test_', '', 1) not in args.testmodules:
367                    continue
368            tests_to_run.append(t)
369
370    if args.update_tests_db:
371        for t in tests_to_run:
372            name = t.__name__.replace('test_', '', 1)
373            print(name + " - " + get_test_description(t))
374            if conn:
375                sql = 'INSERT OR REPLACE INTO tests(test,description) VALUES (?, ?)'
376                params = (name, get_test_description(t))
377                try:
378                    conn.execute(sql, params)
379                except Exception as e:
380                    print("sqlite: " + str(e))
381                    print("sql: %r" % (params,))
382        if conn:
383            conn.commit()
384            conn.close()
385        sys.exit(0)
386
387    if not args.logdir:
388        if os.path.exists('logs/current'):
389            args.logdir = 'logs/current'
390        else:
391            args.logdir = 'logs'
392
393    # Write debug level log to a file and configurable verbosity to stdout
394    logger.setLevel(logging.DEBUG)
395
396    stdout_handler = logging.StreamHandler()
397    stdout_handler.setLevel(args.loglevel)
398    logger.addHandler(stdout_handler)
399
400    file_name = os.path.join(args.logdir, 'run-tests.log')
401    log_handler = logging.FileHandler(file_name, encoding='utf-8')
402    log_handler.setLevel(logging.DEBUG)
403    fmt = "%(asctime)s %(levelname)s %(message)s"
404    log_formatter = logging.Formatter(fmt)
405    log_handler.setFormatter(log_formatter)
406    logger.addHandler(log_handler)
407
408    dev0 = WpaSupplicant('wlan0', '/tmp/wpas-wlan0')
409    dev1 = WpaSupplicant('wlan1', '/tmp/wpas-wlan1')
410    dev2 = WpaSupplicant('wlan2', '/tmp/wpas-wlan2')
411    dev = [dev0, dev1, dev2]
412    apdev = []
413    apdev.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"})
414    apdev.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"})
415
416    for d in dev:
417        if not d.ping():
418            logger.info(d.ifname + ": No response from wpa_supplicant")
419            return
420        logger.info("DEV: " + d.ifname + ": " + d.p2p_dev_addr())
421    for ap in apdev:
422        logger.info("APDEV: " + ap['ifname'])
423
424    passed = []
425    skipped = []
426    failed = []
427
428    # make sure nothing is left over from previous runs
429    # (if there were any other manual runs or we crashed)
430    if not reset_devs(dev, apdev):
431        if conn:
432            conn.close()
433            conn = None
434        sys.exit(1)
435
436    if args.dmesg:
437        subprocess.call(['dmesg', '-c'], stdout=open('/dev/null', 'w'))
438
439    try:
440        # try to clear out any leaks that happened earlier
441        with open('/sys/kernel/debug/kmemleak', 'w') as kmemleak:
442            kmemleak.write('scan')
443            kmemleak.seek(0)
444            time.sleep(5)
445            kmemleak.write('scan')
446            kmemleak.seek(0)
447            kmemleak.write('clear')
448        have_kmemleak = True
449    except OSError:
450        have_kmemleak = False
451
452    if conn and args.prefill:
453        for t in tests_to_run:
454            name = t.__name__.replace('test_', '', 1)
455            report(conn, False, args.build, args.commit, run, name, 'NOTRUN', 0,
456                   args.logdir, sql_commit=False)
457        conn.commit()
458
459    if args.split:
460        vals = args.split.split('/')
461        split_server = int(vals[0])
462        split_total = int(vals[1])
463        logger.info("Parallel execution - %d/%d" % (split_server, split_total))
464        split_server -= 1
465        tests_to_run.sort(key=lambda t: t.__name__)
466        tests_to_run = [x for i, x in enumerate(tests_to_run) if i % split_total == split_server]
467
468    if args.shuffle_tests:
469        from random import shuffle
470        shuffle(tests_to_run)
471
472    count = 0
473    if args.stdin_ctrl:
474        print("READY")
475        sys.stdout.flush()
476        num_tests = 0
477    else:
478        num_tests = len(tests_to_run)
479    if args.stdin_ctrl:
480        set_term_echo(sys.stdin.fileno(), False)
481
482    check_country_00 = True
483    for d in dev:
484        if d.get_driver_status_field("country") != "00":
485            check_country_00 = False
486
487    while True:
488        if args.stdin_ctrl:
489            test = sys.stdin.readline()
490            if not test:
491                break
492            test = test.splitlines()[0]
493            if test == '':
494                break
495            t = None
496            for tt in tests:
497                name = tt.__name__.replace('test_', '', 1)
498                if name == test:
499                    t = tt
500                    break
501            if not t:
502                print("NOT-FOUND")
503                sys.stdout.flush()
504                continue
505        else:
506            if len(tests_to_run) == 0:
507                break
508            t = tests_to_run.pop(0)
509
510        if dev[0].get_driver_status_field("country") == "98":
511            # Work around cfg80211 regulatory issues in clearing intersected
512            # country code 98. Need to make station disconnect without any
513            # other wiphy being active in the system.
514            logger.info("country=98 workaround - try to clear state")
515            id = dev[1].add_network()
516            dev[1].set_network(id, "mode", "2")
517            dev[1].set_network_quoted(id, "ssid", "country98")
518            dev[1].set_network(id, "key_mgmt", "NONE")
519            dev[1].set_network(id, "frequency", "2412")
520            dev[1].set_network(id, "scan_freq", "2412")
521            dev[1].select_network(id)
522            ev = dev[1].wait_event(["CTRL-EVENT-CONNECTED"])
523            if ev:
524                dev[0].connect("country98", key_mgmt="NONE", scan_freq="2412")
525                dev[1].request("DISCONNECT")
526                dev[0].wait_disconnected()
527                dev[0].disconnect_and_stop_scan()
528            dev[0].reset()
529            dev[1].reset()
530            dev[0].dump_monitor()
531            dev[1].dump_monitor()
532
533        name = t.__name__.replace('test_', '', 1)
534        open('/dev/kmsg', 'w').write('running hwsim test case %s\n' % name)
535        if log_handler:
536            log_handler.stream.close()
537            logger.removeHandler(log_handler)
538            file_name = os.path.join(args.logdir, name + '.log')
539            log_handler = logging.FileHandler(file_name, encoding='utf-8')
540            log_handler.setLevel(logging.DEBUG)
541            log_handler.setFormatter(log_formatter)
542            logger.addHandler(log_handler)
543
544        try:
545            with open('/sys/kernel/debug/clear_warn_once', 'w') as f:
546                f.write('1\n')
547        except FileNotFoundError:
548            pass
549
550        reset_ok = True
551        with DataCollector(args.logdir, name, have_kmemleak, args):
552            count = count + 1
553            msg = "START {} {}/{}".format(name, count, num_tests)
554            logger.info(msg)
555            if args.loglevel == logging.WARNING:
556                print(msg)
557                sys.stdout.flush()
558            if t.__doc__:
559                logger.info("Test: " + t.__doc__)
560            start = datetime.now()
561            open('/dev/kmsg', 'w').write('TEST-START %s @%.6f\n' % (name, time.time()))
562            for d in dev:
563                try:
564                    d.dump_monitor()
565                    if not d.ping():
566                        raise Exception("PING failed for {}".format(d.ifname))
567                    if not d.global_ping():
568                        raise Exception("Global PING failed for {}".format(d.ifname))
569                    d.request("NOTE TEST-START " + name)
570                except Exception as e:
571                    logger.info("Failed to issue TEST-START before " + name + " for " + d.ifname)
572                    logger.info(e)
573                    print("FAIL " + name + " - could not start test")
574                    if conn:
575                        conn.close()
576                        conn = None
577                    if args.stdin_ctrl:
578                        set_term_echo(sys.stdin.fileno(), True)
579                    sys.exit(1)
580            skip_reason = None
581            try:
582                if is_long_duration_test(t) and not args.long:
583                    raise HwsimSkip("Skip test case with long duration due to --long not specified")
584                if t.__code__.co_argcount > 2:
585                    params = {}
586                    params['logdir'] = args.logdir
587                    params['name'] = name
588                    params['prefix'] = os.path.join(args.logdir, name)
589                    t(dev, apdev, params)
590                elif t.__code__.co_argcount > 1:
591                    t(dev, apdev)
592                else:
593                    t(dev)
594                result = "PASS"
595                if check_country_00:
596                    for d in dev:
597                        country = d.get_driver_status_field("country")
598                        if country is None:
599                            logger.info(d.ifname + ": Could not fetch country code after the test case run")
600                        elif country != "00":
601                            d.dump_monitor()
602                            logger.info(d.ifname + ": Country code not reset back to 00: is " + country)
603                            print(d.ifname + ": Country code not reset back to 00: is " + country)
604                            result = "FAIL"
605
606                            # Try to wait for cfg80211 regulatory state to
607                            # clear.
608                            d.cmd_execute(['iw', 'reg', 'set', '00'])
609                            for i in range(5):
610                                time.sleep(1)
611                                country = d.get_driver_status_field("country")
612                                if country == "00":
613                                    break
614                            if country == "00":
615                                print(d.ifname + ": Country code cleared back to 00")
616                                logger.info(d.ifname + ": Country code cleared back to 00")
617                            else:
618                                print("Country code remains set - expect following test cases to fail")
619                                logger.info("Country code remains set - expect following test cases to fail")
620                            break
621            except HwsimSkip as e:
622                logger.info("Skip test case: %s" % e)
623                skip_reason = e
624                result = "SKIP"
625            except NameError as e:
626                import traceback
627                logger.info(e)
628                traceback.print_exc()
629                result = "FAIL"
630            except Exception as e:
631                import traceback
632                logger.info(e)
633                traceback.print_exc()
634                if args.loglevel == logging.WARNING:
635                    print("Exception: " + str(e))
636                result = "FAIL"
637
638            # Work around some objects having __del__, we really should
639            # use context managers, but that's complex. Doing this here
640            # will (on cpython at least) at make sure those objects that
641            # are no longer reachable will be collected now, invoking
642            # __del__() on them. This then ensures that __del__() isn't
643            # invoked at a bad time, e.g. causing recursion in locking.
644            gc.collect()
645
646            open('/dev/kmsg', 'w').write('TEST-STOP %s @%.6f\n' % (name, time.time()))
647            for d in dev:
648                try:
649                    d.dump_monitor()
650                    d.request("NOTE TEST-STOP " + name)
651                except Exception as e:
652                    logger.info("Failed to issue TEST-STOP after {} for {}".format(name, d.ifname))
653                    logger.info(e)
654                    result = "FAIL"
655            if args.no_reset:
656                print("Leaving devices in current state")
657            else:
658                reset_ok = reset_devs(dev, apdev)
659            wpas = None
660            try:
661                wpas = WpaSupplicant(global_iface="/tmp/wpas-wlan5",
662                                     monitor=False)
663                rename_log(args.logdir, 'log5', name, wpas)
664                if not args.no_reset:
665                    wpas.remove_ifname()
666            except Exception as e:
667                pass
668            if wpas:
669                wpas.close_ctrl()
670                del wpas
671
672            for i in range(0, 3):
673                rename_log(args.logdir, 'log' + str(i), name, dev[i])
674            try:
675                hapd = HostapdGlobal()
676            except Exception as e:
677                print("Failed to connect to hostapd interface")
678                print(str(e))
679                reset_ok = False
680                result = "FAIL"
681                hapd = None
682            rename_log(args.logdir, 'hostapd', name, hapd)
683            if hapd:
684                del hapd
685                hapd = None
686
687            # Use None here since this instance of Wlantest() will never be
688            # used for remote host hwsim tests on real hardware.
689            Wlantest.setup(None)
690            wt = Wlantest()
691            rename_log(args.logdir, 'hwsim0.pcapng', name, wt)
692            rename_log(args.logdir, 'hwsim0', name, wt)
693            if os.path.exists(os.path.join(args.logdir, 'fst-wpa_supplicant')):
694                rename_log(args.logdir, 'fst-wpa_supplicant', name, None)
695            if os.path.exists(os.path.join(args.logdir, 'fst-hostapd')):
696                rename_log(args.logdir, 'fst-hostapd', name, None)
697            if os.path.exists(os.path.join(args.logdir, 'wmediumd.log')):
698                rename_log(args.logdir, 'wmediumd.log', name, None)
699
700        end = datetime.now()
701        diff = end - start
702
703        if result == 'PASS' and args.dmesg:
704            if not check_kernel(os.path.join(args.logdir, name + '.dmesg')):
705                logger.info("Kernel issue found in dmesg - mark test failed")
706                result = 'FAIL'
707
708        if result == 'PASS' and have_kmemleak:
709            # The file is only created if a leak was found
710            if os.path.exists(os.path.join(args.logdir, name + '.kmemleak')):
711                logger.info("Kernel memory leak found - mark test failed")
712                result = 'FAIL'
713
714        if result == 'PASS':
715            passed.append(name)
716        elif result == 'SKIP':
717            skipped.append(name)
718        else:
719            failed.append(name)
720
721        report(conn, args.prefill, args.build, args.commit, run, name, result,
722               diff.total_seconds(), args.logdir)
723        result = "{} {} {} {}".format(result, name, diff.total_seconds(), end)
724        logger.info(result)
725        if args.loglevel == logging.WARNING:
726            print(result)
727            if skip_reason:
728                print("REASON", skip_reason)
729            sys.stdout.flush()
730
731        if not reset_ok:
732            print("Terminating early due to device reset failure")
733            break
734
735    for d in dev:
736        d.close_ctrl()
737
738    if args.stdin_ctrl:
739        set_term_echo(sys.stdin.fileno(), True)
740
741    if log_handler:
742        log_handler.stream.close()
743        logger.removeHandler(log_handler)
744        file_name = os.path.join(args.logdir, 'run-tests.log')
745        log_handler = logging.FileHandler(file_name, encoding='utf-8')
746        log_handler.setLevel(logging.DEBUG)
747        log_handler.setFormatter(log_formatter)
748        logger.addHandler(log_handler)
749
750    if conn:
751        conn.close()
752
753    if len(failed):
754        logger.info("passed {} test case(s)".format(len(passed)))
755        logger.info("skipped {} test case(s)".format(len(skipped)))
756        logger.info("failed tests: " + ' '.join(failed))
757        if args.loglevel == logging.WARNING:
758            print("failed tests: " + ' '.join(failed))
759        sys.exit(1)
760    logger.info("passed all {} test case(s)".format(len(passed)))
761    if len(skipped):
762        logger.info("skipped {} test case(s)".format(len(skipped)))
763    if args.loglevel == logging.WARNING:
764        print("passed all {} test case(s)".format(len(passed)))
765        if len(skipped):
766            print("skipped {} test case(s)".format(len(skipped)))
767
768if __name__ == "__main__":
769    main()
770