1 #!/usr/bin/env python3
2 #
3 # Remote test case executor
4 # Copyright (c) 2016, Tieto Corporation
5 #
6 # This software may be distributed under the terms of the BSD license.
7 # See README for more details.
8 
9 import os
10 import re
11 import sys
12 import time
13 import traceback
14 import getopt
15 from datetime import datetime
16 from random import shuffle
17 
18 import logging
19 logger = logging.getLogger()
20 
21 scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
22 sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy'))
23 sys.path.append(os.path.join(scriptsdir, '..', 'hwsim'))
24 
25 import wpaspy
26 import config
27 from test_devices import show_devices
28 from test_devices import check_devices
29 from rutils import TestSkip
30 from utils import HwsimSkip
31 from hwsim_wrapper import run_hwsim_test
32 
33 def usage():
34     print("USAGE: " + sys.argv[0] + " -t devices")
35     print("USAGE: " + sys.argv[0] + " -t check_devices")
36     print("USAGE: " + sys.argv[0] + " -d <dut_name> -t <all|sanity|tests_to_run> [-r <ref_name>] [-c <cfg_file.py>] [-m <all|monitor_name>] [-h hwsim_tests] [-f hwsim_modules][-R][-T][-P][-S][-v]")
37     print("USAGE: " + sys.argv[0])
38 
39 def get_devices(devices, duts, refs, monitors):
40     for dut in duts:
41         config.get_device(devices, dut, lock=True)
42     for ref in refs:
43         config.get_device(devices, ref, lock=True)
44     for monitor in monitors:
45         if monitor == "all":
46             continue
47         if monitor in duts:
48             continue
49         if monitor in refs:
50             continue
51         config.get_device(devices, monitor, lock=True)
52 
53 def put_devices(devices, duts, refs, monitors):
54     for dut in duts:
55         config.put_device(devices, dut)
56     for ref in refs:
57         config.put_device(devices, ref)
58     for monitor in monitors:
59         if monitor == "all":
60             continue
61         if monitor in duts:
62             continue
63         if monitor in refs:
64             continue
65         config.put_device(devices, monitor)
66 
67 def main():
68     duts = []
69     refs = []
70     monitors = []
71     filter_keys = []
72     requested_tests = ["help"]
73     requested_hwsim_tests = []
74     hwsim_tests = []
75     requested_modules = []
76     modules_tests = []
77     cfg_file = "cfg.py"
78     log_dir = "./logs/"
79     verbose = False
80     trace = False
81     restart = False
82     perf = False
83     shuffle_tests = False
84 
85     # parse input parameters
86     try:
87         opts, args = getopt.getopt(sys.argv[1:], "d:f:r:t:l:k:c:m:h:vRPTS",
88                                    ["dut=", "modules=", "ref=", "tests=",
89                                     "log-dir=",
90                                     "cfg=", "key=", "monitor=", "hwsim="])
91     except getopt.GetoptError as err:
92         print(err)
93         usage()
94         sys.exit(2)
95 
96     for option, argument in opts:
97         if option == "-v":
98             verbose = True
99         elif option == "-R":
100             restart = True
101         elif option == "-T":
102             trace = True
103         elif option == "-P":
104             perf = True
105         elif option == "-S":
106             shuffle_tests = True
107         elif option in ("-d", "--dut"):
108             duts.append(argument)
109         elif option in ("-r", "--ref"):
110             refs.append(argument)
111         elif option in ("-t", "--tests"):
112             requested_tests = re.split('; | |, ', argument)
113         elif option in ("-l", "--log-dir"):
114             log_dir = argument
115         elif option in ("-k", "--key"):
116             filter_keys.append(argument)
117         elif option in ("-m", "--monitor"):
118             monitors.append(argument)
119         elif option in ("-c", "--cfg"):
120             cfg_file = argument
121         elif option in ("-h", "--hwsim"):
122             requested_hwsim_tests = re.split('; | |, ', argument)
123         elif option in ("-f", "--modules"):
124             requested_modules = re.split('; | |, ', argument)
125         else:
126             assert False, "unhandled option"
127 
128     # get env configuration
129     setup_params = config.get_setup_params(cfg_file)
130     devices = config.get_devices(cfg_file)
131 
132     # put logs in log_dir
133     symlink = os.path.join(log_dir, "current");
134     if os.path.exists(symlink):
135         os.unlink(symlink)
136     log_dir = os.path.join(log_dir, time.strftime("%Y_%m_%d_%H_%M_%S"))
137     if not os.path.exists(log_dir):
138         os.makedirs(log_dir)
139     os.symlink(os.path.join("../", log_dir), symlink)
140 
141     # setup restart/trace/perf request
142     setup_params['local_log_dir'] = log_dir
143     setup_params['restart_device'] = restart
144     setup_params['trace'] = trace
145     setup_params['perf'] = perf
146 
147     # configure logger
148     logger.setLevel(logging.DEBUG)
149 
150     stdout_handler = logging.StreamHandler()
151     stdout_handler.setLevel(logging.WARNING)
152     if verbose:
153         stdout_handler.setLevel(logging.DEBUG)
154     logger.addHandler(stdout_handler)
155 
156     formatter = logging.Formatter('%(asctime)s - %(message)s')
157     file_name = os.path.join(log_dir, 'run-tests.log')
158     log_handler = logging.FileHandler(file_name)
159     log_handler.setLevel(logging.DEBUG)
160     log_handler.setFormatter(formatter)
161     logger.addHandler(log_handler)
162 
163     # import available tests
164     tests = []
165     failed = []
166     test_modules = []
167     files = os.listdir(scriptsdir)
168     for t in files:
169         m = re.match(r'(test_.*)\.py$', t)
170         if m:
171             mod = __import__(m.group(1))
172             test_modules.append(mod.__name__.replace('test_', '', 1))
173             for key, val in mod.__dict__.items():
174                 if key.startswith("test_"):
175                     tests.append(val)
176     test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests]))
177 
178     # import test_*
179     files = os.listdir("../hwsim/")
180     for t in files:
181         m = re.match(r'(test_.*)\.py$', t)
182         if m:
183             mod = __import__(m.group(1))
184             test_modules.append(mod.__name__.replace('test_', '', 1))
185             for key, val in mod.__dict__.items():
186                 if key.startswith("test_"):
187                     hwsim_tests.append(val)
188 
189     # setup hwsim tests
190     hwsim_tests_to_run = []
191     if len(requested_hwsim_tests) > 0:
192         # apply filters
193         for filter_key in filter_keys:
194             filtered_tests = []
195             for hwsim_test in hwsim_tests:
196                 if re.search(filter_key, hwsim_test.__name__):
197                     filtered_tests.append(hwsim_test)
198             hwsim_tests = filtered_tests
199 
200         # setup hwsim_test we should run
201         if requested_hwsim_tests[0] == "all":
202             hwsim_tests_to_run = hwsim_tests
203         elif requested_hwsim_tests[0] == "remote":
204             hwsim_tests_to_run = [t for t in hwsim_tests
205                                   if hasattr(t, "remote_compatible") and
206                                      t.remote_compatible]
207         else:
208             for test in requested_hwsim_tests:
209                 t = None
210                 for tt in hwsim_tests:
211                     name = tt.__name__.replace('test_', '', 1)
212                     if name == test:
213                         t = tt
214                         break
215                 if not t:
216                     logger.warning("hwsim test case: " + test + " NOT-FOUND")
217                     continue
218                 hwsim_tests_to_run.append(t)
219 
220     # import test_* from modules
221     files = os.listdir("../hwsim/")
222     for t in files:
223         m = re.match(r'(test_.*)\.py$', t)
224         if m:
225             mod = __import__(m.group(1))
226             if mod.__name__.replace('test_', '', 1) not in requested_modules:
227                 continue
228             for key, val in mod.__dict__.items():
229                 if key.startswith("test_"):
230                     modules_tests.append(val)
231 
232     if len(requested_modules) > 0:
233         requested_hwsim_tests = modules_tests
234         hwsim_tests_to_run = modules_tests
235 
236     # sort the list
237     test_names.sort()
238     tests.sort(key=lambda t: t.__name__)
239 
240     # print help
241     if requested_tests[0] == "help" and len(requested_hwsim_tests) == 0:
242         usage()
243         print("\nAvailable Devices:")
244         for device in devices:
245             print("\t", device['name'])
246         print("\nAvailable tests:")
247         for test in test_names:
248             print("\t", test)
249         print("\nAvailable hwsim tests:")
250         for hwsim_test in hwsim_tests:
251             print("\t", hwsim_test.__name__.replace('test_', '', 1))
252         return
253 
254     # show/check devices
255     if requested_tests[0] == "devices":
256         show_devices(devices, setup_params)
257         return
258 
259     # apply filters
260     for filter_key in filter_keys:
261         filtered_tests = []
262         for test in tests:
263             if re.search(filter_key, test.__name__):
264                 filtered_tests.append(test)
265         tests = filtered_tests
266 
267     # setup test we should run
268     tests_to_run = []
269     if requested_tests[0] == "all":
270         tests_to_run = tests
271     if requested_tests[0] == "help":
272         pass
273     elif requested_tests[0] == "sanity":
274         for test in tests:
275             if test.__name__.startswith("test_sanity_"):
276                 tests_to_run.append(test)
277     else:
278         for test in requested_tests:
279             t = None
280             for tt in tests:
281                 name = tt.__name__.replace('test_', '', 1)
282                 if name == test:
283                     t = tt
284                     break
285             if not t:
286                 logger.warning("test case: " + test + " NOT-FOUND")
287                 continue
288             tests_to_run.append(t)
289 
290     if shuffle_tests:
291         shuffle(tests_to_run)
292         shuffle(hwsim_tests_to_run)
293 
294     # lock devices
295     try:
296         get_devices(devices, duts, refs, monitors)
297     except Exception as e:
298         logger.warning("get devices failed: " + str(e))
299         logger.info(traceback.format_exc())
300         put_devices(devices, duts, refs, monitors)
301         return
302     except:
303         logger.warning("get devices failed")
304         logger.info(traceback.format_exc())
305         put_devices(devices, duts, refs, monitors)
306         return
307 
308     # now run test cases
309     for dut in duts:
310         if len(requested_hwsim_tests) > 0:
311             logger.warning("DUT (apdev): " + str(dut))
312         else:
313             logger.warning("DUT: " + str(dut))
314     for ref in refs:
315         if len(requested_hwsim_tests) > 0:
316             logger.warning("REF   (dev): " + str(ref))
317         else:
318             logger.warning("REF: " + str(ref))
319     for monitor in monitors:
320         logger.warning("MON: " + str(monitor))
321 
322     # run check_devices at beginning
323     logger.warning("RUN check_devices")
324     try:
325         check_devices(devices, setup_params, refs, duts, monitors)
326     except Exception as e:
327         logger.warning("FAILED: " + str(e))
328         logger.info(traceback.format_exc())
329         put_devices(devices, duts, refs, monitors)
330         return
331     except:
332         logger.warning("FAILED")
333         logger.info(traceback.format_exc())
334         put_devices(devices, duts, refs, monitors)
335         return
336     logger.warning("PASS")
337 
338     test_no = 1
339     for test in tests_to_run:
340         try:
341             start = datetime.now()
342             setup_params['tc_name'] = test.__name__.replace('test_', '', 1)
343             logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(tests_to_run)) + ")")
344             if test.__doc__:
345                 logger.info("Test: " + test.__doc__)
346 
347             # run tc
348             res = test(devices, setup_params, refs, duts, monitors)
349 
350             end = datetime.now()
351             logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
352         except KeyboardInterrupt:
353             put_devices(devices, duts, refs, monitors)
354             raise
355         except TestSkip as e:
356             end = datetime.now()
357             logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
358         except Exception as e:
359             end = datetime.now()
360             logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
361             logger.info(traceback.format_exc())
362             failed.append(test.__name__.replace('test_', '', 1))
363         except:
364             end = datetime.now()
365             logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
366             logger.info(traceback.format_exc())
367             failed.append(test.__name__.replace('test_', '', 1))
368         test_no += 1
369 
370     test_no = 1
371     for hwsim_test in hwsim_tests_to_run:
372         try:
373             start = datetime.now()
374             setup_params['tc_name'] = hwsim_test.__name__.replace('test_', '', 1)
375             logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(hwsim_tests_to_run)) + ")")
376             res = run_hwsim_test(devices, setup_params, refs, duts, monitors, hwsim_test)
377             end = datetime.now()
378             logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
379         except KeyboardInterrupt:
380             put_devices(devices, duts, refs, monitors)
381             raise
382         except HwsimSkip as e:
383             end = datetime.now()
384             logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
385             failed.append(hwsim_test.__name__.replace('test_', '', 1))
386         except Exception as e:
387             end = datetime.now()
388             logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
389             logger.info(traceback.format_exc())
390             failed.append(hwsim_test.__name__.replace('test_', '', 1))
391         except:
392             end = datetime.now()
393             logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
394             logger.info(traceback.format_exc())
395             failed.append(hwsim_test.__name__.replace('test_', '', 1))
396         test_no += 1
397 
398     # unlock devices
399     put_devices(devices, duts, refs, monitors)
400 
401     if len(failed) > 0:
402         logger.warning("Failed test cases:")
403         for test in failed:
404             logger.warning("\t" + test)
405 
406 
407 if __name__ == "__main__":
408         main()
409