1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2021 ARM Limited.
4 * Original author: Mark Brown <broonie@kernel.org>
5 */
6 #include <assert.h>
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <stdbool.h>
10 #include <stddef.h>
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <string.h>
14 #include <unistd.h>
15 #include <sys/auxv.h>
16 #include <sys/prctl.h>
17 #include <sys/types.h>
18 #include <sys/wait.h>
19 #include <asm/sigcontext.h>
20 #include <asm/hwcap.h>
21
22 #include "../../kselftest.h"
23 #include "rdvl.h"
24
25 #define ARCH_MIN_VL SVE_VL_MIN
26
27 struct vec_data {
28 const char *name;
29 unsigned long hwcap_type;
30 unsigned long hwcap;
31 const char *rdvl_binary;
32 int (*rdvl)(void);
33
34 int prctl_get;
35 int prctl_set;
36 const char *default_vl_file;
37
38 int default_vl;
39 int min_vl;
40 int max_vl;
41 };
42
43 #define VEC_SVE 0
44 #define VEC_SME 1
45
46 static struct vec_data vec_data[] = {
47 [VEC_SVE] = {
48 .name = "SVE",
49 .hwcap_type = AT_HWCAP,
50 .hwcap = HWCAP_SVE,
51 .rdvl = rdvl_sve,
52 .rdvl_binary = "./rdvl-sve",
53 .prctl_get = PR_SVE_GET_VL,
54 .prctl_set = PR_SVE_SET_VL,
55 .default_vl_file = "/proc/sys/abi/sve_default_vector_length",
56 },
57 [VEC_SME] = {
58 .name = "SME",
59 .hwcap_type = AT_HWCAP2,
60 .hwcap = HWCAP2_SME,
61 .rdvl = rdvl_sme,
62 .rdvl_binary = "./rdvl-sme",
63 .prctl_get = PR_SME_GET_VL,
64 .prctl_set = PR_SME_SET_VL,
65 .default_vl_file = "/proc/sys/abi/sme_default_vector_length",
66 },
67 };
68
vec_type_supported(struct vec_data * data)69 static bool vec_type_supported(struct vec_data *data)
70 {
71 return getauxval(data->hwcap_type) & data->hwcap;
72 }
73
stdio_read_integer(FILE * f,const char * what,int * val)74 static int stdio_read_integer(FILE *f, const char *what, int *val)
75 {
76 int n = 0;
77 int ret;
78
79 ret = fscanf(f, "%d%*1[\n]%n", val, &n);
80 if (ret < 1 || n < 1) {
81 ksft_print_msg("failed to parse integer from %s\n", what);
82 return -1;
83 }
84
85 return 0;
86 }
87
88 /* Start a new process and return the vector length it sees */
get_child_rdvl(struct vec_data * data)89 static int get_child_rdvl(struct vec_data *data)
90 {
91 FILE *out;
92 int pipefd[2];
93 pid_t pid, child;
94 int read_vl, ret;
95
96 ret = pipe(pipefd);
97 if (ret == -1) {
98 ksft_print_msg("pipe() failed: %d (%s)\n",
99 errno, strerror(errno));
100 return -1;
101 }
102
103 fflush(stdout);
104
105 child = fork();
106 if (child == -1) {
107 ksft_print_msg("fork() failed: %d (%s)\n",
108 errno, strerror(errno));
109 close(pipefd[0]);
110 close(pipefd[1]);
111 return -1;
112 }
113
114 /* Child: put vector length on the pipe */
115 if (child == 0) {
116 /*
117 * Replace stdout with the pipe, errors to stderr from
118 * here as kselftest prints to stdout.
119 */
120 ret = dup2(pipefd[1], 1);
121 if (ret == -1) {
122 fprintf(stderr, "dup2() %d\n", errno);
123 exit(EXIT_FAILURE);
124 }
125
126 /* exec() a new binary which puts the VL on stdout */
127 ret = execl(data->rdvl_binary, data->rdvl_binary, NULL);
128 fprintf(stderr, "execl(%s) failed: %d (%s)\n",
129 data->rdvl_binary, errno, strerror(errno));
130
131 exit(EXIT_FAILURE);
132 }
133
134 close(pipefd[1]);
135
136 /* Parent; wait for the exit status from the child & verify it */
137 do {
138 pid = wait(&ret);
139 if (pid == -1) {
140 ksft_print_msg("wait() failed: %d (%s)\n",
141 errno, strerror(errno));
142 close(pipefd[0]);
143 return -1;
144 }
145 } while (pid != child);
146
147 assert(pid == child);
148
149 if (!WIFEXITED(ret)) {
150 ksft_print_msg("child exited abnormally\n");
151 close(pipefd[0]);
152 return -1;
153 }
154
155 if (WEXITSTATUS(ret) != 0) {
156 ksft_print_msg("child returned error %d\n",
157 WEXITSTATUS(ret));
158 close(pipefd[0]);
159 return -1;
160 }
161
162 out = fdopen(pipefd[0], "r");
163 if (!out) {
164 ksft_print_msg("failed to open child stdout\n");
165 close(pipefd[0]);
166 return -1;
167 }
168
169 ret = stdio_read_integer(out, "child", &read_vl);
170 fclose(out);
171 if (ret != 0)
172 return ret;
173
174 return read_vl;
175 }
176
file_read_integer(const char * name,int * val)177 static int file_read_integer(const char *name, int *val)
178 {
179 FILE *f;
180 int ret;
181
182 f = fopen(name, "r");
183 if (!f) {
184 ksft_test_result_fail("Unable to open %s: %d (%s)\n",
185 name, errno,
186 strerror(errno));
187 return -1;
188 }
189
190 ret = stdio_read_integer(f, name, val);
191 fclose(f);
192
193 return ret;
194 }
195
file_write_integer(const char * name,int val)196 static int file_write_integer(const char *name, int val)
197 {
198 FILE *f;
199
200 f = fopen(name, "w");
201 if (!f) {
202 ksft_test_result_fail("Unable to open %s: %d (%s)\n",
203 name, errno,
204 strerror(errno));
205 return -1;
206 }
207
208 fprintf(f, "%d", val);
209 fclose(f);
210
211 return 0;
212 }
213
214 /*
215 * Verify that we can read the default VL via proc, checking that it
216 * is set in a freshly spawned child.
217 */
proc_read_default(struct vec_data * data)218 static void proc_read_default(struct vec_data *data)
219 {
220 int default_vl, child_vl, ret;
221
222 ret = file_read_integer(data->default_vl_file, &default_vl);
223 if (ret != 0)
224 return;
225
226 /* Is this the actual default seen by new processes? */
227 child_vl = get_child_rdvl(data);
228 if (child_vl != default_vl) {
229 ksft_test_result_fail("%s is %d but child VL is %d\n",
230 data->default_vl_file,
231 default_vl, child_vl);
232 return;
233 }
234
235 ksft_test_result_pass("%s default vector length %d\n", data->name,
236 default_vl);
237 data->default_vl = default_vl;
238 }
239
240 /* Verify that we can write a minimum value and have it take effect */
proc_write_min(struct vec_data * data)241 static void proc_write_min(struct vec_data *data)
242 {
243 int ret, new_default, child_vl;
244
245 if (geteuid() != 0) {
246 ksft_test_result_skip("Need to be root to write to /proc\n");
247 return;
248 }
249
250 ret = file_write_integer(data->default_vl_file, ARCH_MIN_VL);
251 if (ret != 0)
252 return;
253
254 /* What was the new value? */
255 ret = file_read_integer(data->default_vl_file, &new_default);
256 if (ret != 0)
257 return;
258
259 /* Did it take effect in a new process? */
260 child_vl = get_child_rdvl(data);
261 if (child_vl != new_default) {
262 ksft_test_result_fail("%s is %d but child VL is %d\n",
263 data->default_vl_file,
264 new_default, child_vl);
265 return;
266 }
267
268 ksft_test_result_pass("%s minimum vector length %d\n", data->name,
269 new_default);
270 data->min_vl = new_default;
271
272 file_write_integer(data->default_vl_file, data->default_vl);
273 }
274
275 /* Verify that we can write a maximum value and have it take effect */
proc_write_max(struct vec_data * data)276 static void proc_write_max(struct vec_data *data)
277 {
278 int ret, new_default, child_vl;
279
280 if (geteuid() != 0) {
281 ksft_test_result_skip("Need to be root to write to /proc\n");
282 return;
283 }
284
285 /* -1 is accepted by the /proc interface as the maximum VL */
286 ret = file_write_integer(data->default_vl_file, -1);
287 if (ret != 0)
288 return;
289
290 /* What was the new value? */
291 ret = file_read_integer(data->default_vl_file, &new_default);
292 if (ret != 0)
293 return;
294
295 /* Did it take effect in a new process? */
296 child_vl = get_child_rdvl(data);
297 if (child_vl != new_default) {
298 ksft_test_result_fail("%s is %d but child VL is %d\n",
299 data->default_vl_file,
300 new_default, child_vl);
301 return;
302 }
303
304 ksft_test_result_pass("%s maximum vector length %d\n", data->name,
305 new_default);
306 data->max_vl = new_default;
307
308 file_write_integer(data->default_vl_file, data->default_vl);
309 }
310
311 /* Can we read back a VL from prctl? */
prctl_get(struct vec_data * data)312 static void prctl_get(struct vec_data *data)
313 {
314 int ret;
315
316 ret = prctl(data->prctl_get);
317 if (ret == -1) {
318 ksft_test_result_fail("%s prctl() read failed: %d (%s)\n",
319 data->name, errno, strerror(errno));
320 return;
321 }
322
323 /* Mask out any flags */
324 ret &= PR_SVE_VL_LEN_MASK;
325
326 /* Is that what we can read back directly? */
327 if (ret == data->rdvl())
328 ksft_test_result_pass("%s current VL is %d\n",
329 data->name, ret);
330 else
331 ksft_test_result_fail("%s prctl() VL %d but RDVL is %d\n",
332 data->name, ret, data->rdvl());
333 }
334
335 /* Does the prctl let us set the VL we already have? */
prctl_set_same(struct vec_data * data)336 static void prctl_set_same(struct vec_data *data)
337 {
338 int cur_vl = data->rdvl();
339 int ret;
340
341 ret = prctl(data->prctl_set, cur_vl);
342 if (ret < 0) {
343 ksft_test_result_fail("%s prctl set failed: %d (%s)\n",
344 data->name, errno, strerror(errno));
345 return;
346 }
347
348 ksft_test_result(cur_vl == data->rdvl(),
349 "%s set VL %d and have VL %d\n",
350 data->name, cur_vl, data->rdvl());
351 }
352
353 /* Can we set a new VL for this process? */
prctl_set(struct vec_data * data)354 static void prctl_set(struct vec_data *data)
355 {
356 int ret;
357
358 if (data->min_vl == data->max_vl) {
359 ksft_test_result_skip("%s only one VL supported\n",
360 data->name);
361 return;
362 }
363
364 /* Try to set the minimum VL */
365 ret = prctl(data->prctl_set, data->min_vl);
366 if (ret < 0) {
367 ksft_test_result_fail("%s prctl set failed for %d: %d (%s)\n",
368 data->name, data->min_vl,
369 errno, strerror(errno));
370 return;
371 }
372
373 if ((ret & PR_SVE_VL_LEN_MASK) != data->min_vl) {
374 ksft_test_result_fail("%s prctl set %d but return value is %d\n",
375 data->name, data->min_vl, data->rdvl());
376 return;
377 }
378
379 if (data->rdvl() != data->min_vl) {
380 ksft_test_result_fail("%s set %d but RDVL is %d\n",
381 data->name, data->min_vl, data->rdvl());
382 return;
383 }
384
385 /* Try to set the maximum VL */
386 ret = prctl(data->prctl_set, data->max_vl);
387 if (ret < 0) {
388 ksft_test_result_fail("%s prctl set failed for %d: %d (%s)\n",
389 data->name, data->max_vl,
390 errno, strerror(errno));
391 return;
392 }
393
394 if ((ret & PR_SVE_VL_LEN_MASK) != data->max_vl) {
395 ksft_test_result_fail("%s prctl() set %d but return value is %d\n",
396 data->name, data->max_vl, data->rdvl());
397 return;
398 }
399
400 /* The _INHERIT flag should not be present when we read the VL */
401 ret = prctl(data->prctl_get);
402 if (ret == -1) {
403 ksft_test_result_fail("%s prctl() read failed: %d (%s)\n",
404 data->name, errno, strerror(errno));
405 return;
406 }
407
408 if (ret & PR_SVE_VL_INHERIT) {
409 ksft_test_result_fail("%s prctl() reports _INHERIT\n",
410 data->name);
411 return;
412 }
413
414 ksft_test_result_pass("%s prctl() set min/max\n", data->name);
415 }
416
417 /* If we didn't request it a new VL shouldn't affect the child */
prctl_set_no_child(struct vec_data * data)418 static void prctl_set_no_child(struct vec_data *data)
419 {
420 int ret, child_vl;
421
422 if (data->min_vl == data->max_vl) {
423 ksft_test_result_skip("%s only one VL supported\n",
424 data->name);
425 return;
426 }
427
428 ret = prctl(data->prctl_set, data->min_vl);
429 if (ret < 0) {
430 ksft_test_result_fail("%s prctl set failed for %d: %d (%s)\n",
431 data->name, data->min_vl,
432 errno, strerror(errno));
433 return;
434 }
435
436 /* Ensure the default VL is different */
437 ret = file_write_integer(data->default_vl_file, data->max_vl);
438 if (ret != 0)
439 return;
440
441 /* Check that the child has the default we just set */
442 child_vl = get_child_rdvl(data);
443 if (child_vl != data->max_vl) {
444 ksft_test_result_fail("%s is %d but child VL is %d\n",
445 data->default_vl_file,
446 data->max_vl, child_vl);
447 return;
448 }
449
450 ksft_test_result_pass("%s vector length used default\n", data->name);
451
452 file_write_integer(data->default_vl_file, data->default_vl);
453 }
454
455 /* If we didn't request it a new VL shouldn't affect the child */
prctl_set_for_child(struct vec_data * data)456 static void prctl_set_for_child(struct vec_data *data)
457 {
458 int ret, child_vl;
459
460 if (data->min_vl == data->max_vl) {
461 ksft_test_result_skip("%s only one VL supported\n",
462 data->name);
463 return;
464 }
465
466 ret = prctl(data->prctl_set, data->min_vl | PR_SVE_VL_INHERIT);
467 if (ret < 0) {
468 ksft_test_result_fail("%s prctl set failed for %d: %d (%s)\n",
469 data->name, data->min_vl,
470 errno, strerror(errno));
471 return;
472 }
473
474 /* The _INHERIT flag should be present when we read the VL */
475 ret = prctl(data->prctl_get);
476 if (ret == -1) {
477 ksft_test_result_fail("%s prctl() read failed: %d (%s)\n",
478 data->name, errno, strerror(errno));
479 return;
480 }
481 if (!(ret & PR_SVE_VL_INHERIT)) {
482 ksft_test_result_fail("%s prctl() does not report _INHERIT\n",
483 data->name);
484 return;
485 }
486
487 /* Ensure the default VL is different */
488 ret = file_write_integer(data->default_vl_file, data->max_vl);
489 if (ret != 0)
490 return;
491
492 /* Check that the child inherited our VL */
493 child_vl = get_child_rdvl(data);
494 if (child_vl != data->min_vl) {
495 ksft_test_result_fail("%s is %d but child VL is %d\n",
496 data->default_vl_file,
497 data->min_vl, child_vl);
498 return;
499 }
500
501 ksft_test_result_pass("%s vector length was inherited\n", data->name);
502
503 file_write_integer(data->default_vl_file, data->default_vl);
504 }
505
506 /* _ONEXEC takes effect only in the child process */
prctl_set_onexec(struct vec_data * data)507 static void prctl_set_onexec(struct vec_data *data)
508 {
509 int ret, child_vl;
510
511 if (data->min_vl == data->max_vl) {
512 ksft_test_result_skip("%s only one VL supported\n",
513 data->name);
514 return;
515 }
516
517 /* Set a known value for the default and our current VL */
518 ret = file_write_integer(data->default_vl_file, data->max_vl);
519 if (ret != 0)
520 return;
521
522 ret = prctl(data->prctl_set, data->max_vl);
523 if (ret < 0) {
524 ksft_test_result_fail("%s prctl set failed for %d: %d (%s)\n",
525 data->name, data->min_vl,
526 errno, strerror(errno));
527 return;
528 }
529
530 /* Set a different value for the child to have on exec */
531 ret = prctl(data->prctl_set, data->min_vl | PR_SVE_SET_VL_ONEXEC);
532 if (ret < 0) {
533 ksft_test_result_fail("%s prctl set failed for %d: %d (%s)\n",
534 data->name, data->min_vl,
535 errno, strerror(errno));
536 return;
537 }
538
539 /* Our current VL should stay the same */
540 if (data->rdvl() != data->max_vl) {
541 ksft_test_result_fail("%s VL changed by _ONEXEC prctl()\n",
542 data->name);
543 return;
544 }
545
546 /* Check that the child inherited our VL */
547 child_vl = get_child_rdvl(data);
548 if (child_vl != data->min_vl) {
549 ksft_test_result_fail("Set %d _ONEXEC but child VL is %d\n",
550 data->min_vl, child_vl);
551 return;
552 }
553
554 ksft_test_result_pass("%s vector length set on exec\n", data->name);
555
556 file_write_integer(data->default_vl_file, data->default_vl);
557 }
558
559 /* For each VQ verify that setting via prctl() does the right thing */
prctl_set_all_vqs(struct vec_data * data)560 static void prctl_set_all_vqs(struct vec_data *data)
561 {
562 int ret, vq, vl, new_vl, i;
563 int orig_vls[ARRAY_SIZE(vec_data)];
564 int errors = 0;
565
566 if (!data->min_vl || !data->max_vl) {
567 ksft_test_result_skip("%s Failed to enumerate VLs, not testing VL setting\n",
568 data->name);
569 return;
570 }
571
572 for (i = 0; i < ARRAY_SIZE(vec_data); i++) {
573 if (!vec_type_supported(&vec_data[i]))
574 continue;
575 orig_vls[i] = vec_data[i].rdvl();
576 }
577
578 for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; vq++) {
579 vl = sve_vl_from_vq(vq);
580
581 /* Attempt to set the VL */
582 ret = prctl(data->prctl_set, vl);
583 if (ret < 0) {
584 errors++;
585 ksft_print_msg("%s prctl set failed for %d: %d (%s)\n",
586 data->name, vl,
587 errno, strerror(errno));
588 continue;
589 }
590
591 new_vl = ret & PR_SVE_VL_LEN_MASK;
592
593 /* Check that we actually have the reported new VL */
594 if (data->rdvl() != new_vl) {
595 ksft_print_msg("Set %s VL %d but RDVL reports %d\n",
596 data->name, new_vl, data->rdvl());
597 errors++;
598 }
599
600 /* Did any other VLs change? */
601 for (i = 0; i < ARRAY_SIZE(vec_data); i++) {
602 if (&vec_data[i] == data)
603 continue;
604
605 if (!vec_type_supported(&vec_data[i]))
606 continue;
607
608 if (vec_data[i].rdvl() != orig_vls[i]) {
609 ksft_print_msg("%s VL changed from %d to %d\n",
610 vec_data[i].name, orig_vls[i],
611 vec_data[i].rdvl());
612 errors++;
613 }
614 }
615
616 /* Was that the VL we asked for? */
617 if (new_vl == vl)
618 continue;
619
620 /* Should round up to the minimum VL if below it */
621 if (vl < data->min_vl) {
622 if (new_vl != data->min_vl) {
623 ksft_print_msg("%s VL %d returned %d not minimum %d\n",
624 data->name, vl, new_vl,
625 data->min_vl);
626 errors++;
627 }
628
629 continue;
630 }
631
632 /* Should round down to maximum VL if above it */
633 if (vl > data->max_vl) {
634 if (new_vl != data->max_vl) {
635 ksft_print_msg("%s VL %d returned %d not maximum %d\n",
636 data->name, vl, new_vl,
637 data->max_vl);
638 errors++;
639 }
640
641 continue;
642 }
643
644 /* Otherwise we should've rounded down */
645 if (!(new_vl < vl)) {
646 ksft_print_msg("%s VL %d returned %d, did not round down\n",
647 data->name, vl, new_vl);
648 errors++;
649
650 continue;
651 }
652 }
653
654 ksft_test_result(errors == 0, "%s prctl() set all VLs, %d errors\n",
655 data->name, errors);
656 }
657
658 typedef void (*test_type)(struct vec_data *);
659
660 static const test_type tests[] = {
661 /*
662 * The default/min/max tests must be first and in this order
663 * to provide data for other tests.
664 */
665 proc_read_default,
666 proc_write_min,
667 proc_write_max,
668
669 prctl_get,
670 prctl_set_same,
671 prctl_set,
672 prctl_set_no_child,
673 prctl_set_for_child,
674 prctl_set_onexec,
675 prctl_set_all_vqs,
676 };
677
smstart(void)678 static inline void smstart(void)
679 {
680 asm volatile("msr S0_3_C4_C7_3, xzr");
681 }
682
smstart_sm(void)683 static inline void smstart_sm(void)
684 {
685 asm volatile("msr S0_3_C4_C3_3, xzr");
686 }
687
smstop(void)688 static inline void smstop(void)
689 {
690 asm volatile("msr S0_3_C4_C6_3, xzr");
691 }
692
693
694 /*
695 * Verify we can change the SVE vector length while SME is active and
696 * continue to use SME afterwards.
697 */
change_sve_with_za(void)698 static void change_sve_with_za(void)
699 {
700 struct vec_data *sve_data = &vec_data[VEC_SVE];
701 bool pass = true;
702 int ret, i;
703
704 if (sve_data->min_vl == sve_data->max_vl) {
705 ksft_print_msg("Only one SVE VL supported, can't change\n");
706 ksft_test_result_skip("change_sve_while_sme\n");
707 return;
708 }
709
710 /* Ensure we will trigger a change when we set the maximum */
711 ret = prctl(sve_data->prctl_set, sve_data->min_vl);
712 if (ret != sve_data->min_vl) {
713 ksft_print_msg("Failed to set SVE VL %d: %d\n",
714 sve_data->min_vl, ret);
715 pass = false;
716 }
717
718 /* Enable SM and ZA */
719 smstart();
720
721 /* Trigger another VL change */
722 ret = prctl(sve_data->prctl_set, sve_data->max_vl);
723 if (ret != sve_data->max_vl) {
724 ksft_print_msg("Failed to set SVE VL %d: %d\n",
725 sve_data->max_vl, ret);
726 pass = false;
727 }
728
729 /*
730 * Spin for a bit with SM enabled to try to trigger another
731 * save/restore. We can't use syscalls without exiting
732 * streaming mode.
733 */
734 for (i = 0; i < 100000000; i++)
735 smstart_sm();
736
737 /*
738 * TODO: Verify that ZA was preserved over the VL change and
739 * spin.
740 */
741
742 /* Clean up after ourselves */
743 smstop();
744 ret = prctl(sve_data->prctl_set, sve_data->default_vl);
745 if (ret != sve_data->default_vl) {
746 ksft_print_msg("Failed to restore SVE VL %d: %d\n",
747 sve_data->default_vl, ret);
748 pass = false;
749 }
750
751 ksft_test_result(pass, "change_sve_with_za\n");
752 }
753
754 typedef void (*test_all_type)(void);
755
756 static const struct {
757 const char *name;
758 test_all_type test;
759 } all_types_tests[] = {
760 { "change_sve_with_za", change_sve_with_za },
761 };
762
main(void)763 int main(void)
764 {
765 bool all_supported = true;
766 int i, j;
767
768 ksft_print_header();
769 ksft_set_plan(ARRAY_SIZE(tests) * ARRAY_SIZE(vec_data) +
770 ARRAY_SIZE(all_types_tests));
771
772 for (i = 0; i < ARRAY_SIZE(vec_data); i++) {
773 struct vec_data *data = &vec_data[i];
774 unsigned long supported;
775
776 supported = vec_type_supported(data);
777 if (!supported)
778 all_supported = false;
779
780 for (j = 0; j < ARRAY_SIZE(tests); j++) {
781 if (supported)
782 tests[j](data);
783 else
784 ksft_test_result_skip("%s not supported\n",
785 data->name);
786 }
787 }
788
789 for (i = 0; i < ARRAY_SIZE(all_types_tests); i++) {
790 if (all_supported)
791 all_types_tests[i].test();
792 else
793 ksft_test_result_skip("%s\n", all_types_tests[i].name);
794 }
795
796 ksft_exit_pass();
797 }
798