36 		return 0;  in a6xx_hfi_queue_read()
54 	for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) {  in a6xx_hfi_queue_read()
84 	for (i = 0; i < dwords; i++) {  in a6xx_hfi_queue_write()
92 			queue->data[index] = 0xfafafafa;  in a6xx_hfi_queue_write()
98 	gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01);  in a6xx_hfi_queue_write()
99 	return 0;  in a6xx_hfi_queue_write()
167 		return 0;  in a6xx_hfi_wait_for_ack()
178 	seqnum = atomic_inc_return(&queue->seqnum) % 0xfff;  in a6xx_hfi_send_msg()
196 	struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };  in a6xx_hfi_send_gmu_init()
203 		NULL, 0);  in a6xx_hfi_send_gmu_init()
208 	struct a6xx_hfi_msg_fw_version msg = { 0 };  in a6xx_hfi_get_fw_version()
219 	struct a6xx_hfi_msg_perf_table_v1 msg = { 0 };  in a6xx_hfi_send_perf_table_v1()
225 	for (i = 0; i < gmu->nr_gpu_freqs; i++) {  in a6xx_hfi_send_perf_table_v1()
230 	for (i = 0; i < gmu->nr_gmu_freqs; i++) {  in a6xx_hfi_send_perf_table_v1()
236 		NULL, 0);  in a6xx_hfi_send_perf_table_v1()
241 	struct a6xx_hfi_msg_perf_table msg = { 0 };  in a6xx_hfi_send_perf_table()
247 	for (i = 0; i < gmu->nr_gpu_freqs; i++) {  in a6xx_hfi_send_perf_table()
249 		msg.gx_votes[i].acd = 0xffffffff;  in a6xx_hfi_send_perf_table()
253 	for (i = 0; i < gmu->nr_gmu_freqs; i++) {  in a6xx_hfi_send_perf_table()
259 		NULL, 0);  in a6xx_hfi_send_perf_table()
268 	msg->ddr_wait_bitmask = 0x01;  in a618_build_bw_table()
270 	msg->ddr_cmds_addrs[0] = 0x50000;  in a618_build_bw_table()
271 	msg->ddr_cmds_addrs[1] = 0x5003c;  in a618_build_bw_table()
272 	msg->ddr_cmds_addrs[2] = 0x5000c;  in a618_build_bw_table()
274 	msg->ddr_cmds_data[0][0] =  0x40000000;  in a618_build_bw_table()
275 	msg->ddr_cmds_data[0][1] =  0x40000000;  in a618_build_bw_table()
276 	msg->ddr_cmds_data[0][2] =  0x40000000;  in a618_build_bw_table()
283 	msg->cnoc_wait_bitmask = 0x01;  in a618_build_bw_table()
285 	msg->cnoc_cmds_addrs[0] = 0x5007c;  in a618_build_bw_table()
286 	msg->cnoc_cmds_data[0][0] =  0x40000000;  in a618_build_bw_table()
287 	msg->cnoc_cmds_data[1][0] =  0x60000001;  in a618_build_bw_table()
295 	msg->ddr_wait_bitmask = 0x0;  in a619_build_bw_table()
297 	msg->ddr_cmds_addrs[0] = 0x50000;  in a619_build_bw_table()
298 	msg->ddr_cmds_addrs[1] = 0x50004;  in a619_build_bw_table()
299 	msg->ddr_cmds_addrs[2] = 0x50080;  in a619_build_bw_table()
301 	msg->ddr_cmds_data[0][0]  = 0x40000000;  in a619_build_bw_table()
302 	msg->ddr_cmds_data[0][1]  = 0x40000000;  in a619_build_bw_table()
303 	msg->ddr_cmds_data[0][2]  = 0x40000000;  in a619_build_bw_table()
304 	msg->ddr_cmds_data[1][0]  = 0x6000030c;  in a619_build_bw_table()
305 	msg->ddr_cmds_data[1][1]  = 0x600000db;  in a619_build_bw_table()
306 	msg->ddr_cmds_data[1][2]  = 0x60000008;  in a619_build_bw_table()
307 	msg->ddr_cmds_data[2][0]  = 0x60000618;  in a619_build_bw_table()
308 	msg->ddr_cmds_data[2][1]  = 0x600001b6;  in a619_build_bw_table()
309 	msg->ddr_cmds_data[2][2]  = 0x60000008;  in a619_build_bw_table()
310 	msg->ddr_cmds_data[3][0]  = 0x60000925;  in a619_build_bw_table()
311 	msg->ddr_cmds_data[3][1]  = 0x60000291;  in a619_build_bw_table()
312 	msg->ddr_cmds_data[3][2]  = 0x60000008;  in a619_build_bw_table()
313 	msg->ddr_cmds_data[4][0]  = 0x60000dc1;  in a619_build_bw_table()
314 	msg->ddr_cmds_data[4][1]  = 0x600003dc;  in a619_build_bw_table()
315 	msg->ddr_cmds_data[4][2]  = 0x60000008;  in a619_build_bw_table()
316 	msg->ddr_cmds_data[5][0]  = 0x600010ad;  in a619_build_bw_table()
317 	msg->ddr_cmds_data[5][1]  = 0x600004ae;  in a619_build_bw_table()
318 	msg->ddr_cmds_data[5][2]  = 0x60000008;  in a619_build_bw_table()
319 	msg->ddr_cmds_data[6][0]  = 0x600014c3;  in a619_build_bw_table()
320 	msg->ddr_cmds_data[6][1]  = 0x600005d4;  in a619_build_bw_table()
321 	msg->ddr_cmds_data[6][2]  = 0x60000008;  in a619_build_bw_table()
322 	msg->ddr_cmds_data[7][0]  = 0x6000176a;  in a619_build_bw_table()
323 	msg->ddr_cmds_data[7][1]  = 0x60000693;  in a619_build_bw_table()
324 	msg->ddr_cmds_data[7][2]  = 0x60000008;  in a619_build_bw_table()
325 	msg->ddr_cmds_data[8][0]  = 0x60001f01;  in a619_build_bw_table()
326 	msg->ddr_cmds_data[8][1]  = 0x600008b5;  in a619_build_bw_table()
327 	msg->ddr_cmds_data[8][2]  = 0x60000008;  in a619_build_bw_table()
328 	msg->ddr_cmds_data[9][0]  = 0x60002940;  in a619_build_bw_table()
329 	msg->ddr_cmds_data[9][1]  = 0x60000b95;  in a619_build_bw_table()
330 	msg->ddr_cmds_data[9][2]  = 0x60000008;  in a619_build_bw_table()
331 	msg->ddr_cmds_data[10][0] = 0x60002f68;  in a619_build_bw_table()
332 	msg->ddr_cmds_data[10][1] = 0x60000d50;  in a619_build_bw_table()
333 	msg->ddr_cmds_data[10][2] = 0x60000008;  in a619_build_bw_table()
334 	msg->ddr_cmds_data[11][0] = 0x60003700;  in a619_build_bw_table()
335 	msg->ddr_cmds_data[11][1] = 0x60000f71;  in a619_build_bw_table()
336 	msg->ddr_cmds_data[11][2] = 0x60000008;  in a619_build_bw_table()
337 	msg->ddr_cmds_data[12][0] = 0x60003fce;  in a619_build_bw_table()
338 	msg->ddr_cmds_data[12][1] = 0x600011ea;  in a619_build_bw_table()
339 	msg->ddr_cmds_data[12][2] = 0x60000008;  in a619_build_bw_table()
342 	msg->cnoc_wait_bitmask = 0x0;  in a619_build_bw_table()
344 	msg->cnoc_cmds_addrs[0] = 0x50054;  in a619_build_bw_table()
346 	msg->cnoc_cmds_data[0][0] = 0x40000000;  in a619_build_bw_table()
358 	msg->ddr_wait_bitmask = 0x01;  in a640_build_bw_table()
360 	msg->ddr_cmds_addrs[0] = 0x50000;  in a640_build_bw_table()
361 	msg->ddr_cmds_addrs[1] = 0x5003c;  in a640_build_bw_table()
362 	msg->ddr_cmds_addrs[2] = 0x5000c;  in a640_build_bw_table()
364 	msg->ddr_cmds_data[0][0] =  0x40000000;  in a640_build_bw_table()
365 	msg->ddr_cmds_data[0][1] =  0x40000000;  in a640_build_bw_table()
366 	msg->ddr_cmds_data[0][2] =  0x40000000;  in a640_build_bw_table()
373 	msg->cnoc_wait_bitmask = 0x01;  in a640_build_bw_table()
375 	msg->cnoc_cmds_addrs[0] = 0x50034;  in a640_build_bw_table()
376 	msg->cnoc_cmds_addrs[1] = 0x5007c;  in a640_build_bw_table()
377 	msg->cnoc_cmds_addrs[2] = 0x5004c;  in a640_build_bw_table()
379 	msg->cnoc_cmds_data[0][0] =  0x40000000;  in a640_build_bw_table()
380 	msg->cnoc_cmds_data[0][1] =  0x00000000;  in a640_build_bw_table()
381 	msg->cnoc_cmds_data[0][2] =  0x40000000;  in a640_build_bw_table()
383 	msg->cnoc_cmds_data[1][0] =  0x60000001;  in a640_build_bw_table()
384 	msg->cnoc_cmds_data[1][1] =  0x20000001;  in a640_build_bw_table()
385 	msg->cnoc_cmds_data[1][2] =  0x60000001;  in a640_build_bw_table()
397 	msg->ddr_wait_bitmask = 0x01;  in a650_build_bw_table()
399 	msg->ddr_cmds_addrs[0] = 0x50000;  in a650_build_bw_table()
400 	msg->ddr_cmds_addrs[1] = 0x50004;  in a650_build_bw_table()
401 	msg->ddr_cmds_addrs[2] = 0x5007c;  in a650_build_bw_table()
403 	msg->ddr_cmds_data[0][0] =  0x40000000;  in a650_build_bw_table()
404 	msg->ddr_cmds_data[0][1] =  0x40000000;  in a650_build_bw_table()
405 	msg->ddr_cmds_data[0][2] =  0x40000000;  in a650_build_bw_table()
412 	msg->cnoc_wait_bitmask = 0x01;  in a650_build_bw_table()
414 	msg->cnoc_cmds_addrs[0] = 0x500a4;  in a650_build_bw_table()
415 	msg->cnoc_cmds_data[0][0] =  0x40000000;  in a650_build_bw_table()
416 	msg->cnoc_cmds_data[1][0] =  0x60000001;  in a650_build_bw_table()
428 	msg->ddr_wait_bitmask = 0x01;  in a690_build_bw_table()
430 	msg->ddr_cmds_addrs[0] = 0x50004;  in a690_build_bw_table()
431 	msg->ddr_cmds_addrs[1] = 0x50000;  in a690_build_bw_table()
432 	msg->ddr_cmds_addrs[2] = 0x500ac;  in a690_build_bw_table()
434 	msg->ddr_cmds_data[0][0] =  0x40000000;  in a690_build_bw_table()
435 	msg->ddr_cmds_data[0][1] =  0x40000000;  in a690_build_bw_table()
436 	msg->ddr_cmds_data[0][2] =  0x40000000;  in a690_build_bw_table()
443 	msg->cnoc_wait_bitmask = 0x01;  in a690_build_bw_table()
445 	msg->cnoc_cmds_addrs[0] = 0x5003c;  in a690_build_bw_table()
446 	msg->cnoc_cmds_data[0][0] =  0x40000000;  in a690_build_bw_table()
447 	msg->cnoc_cmds_data[1][0] =  0x60000001;  in a690_build_bw_table()
459 	msg->ddr_wait_bitmask = 0x01;  in a660_build_bw_table()
461 	msg->ddr_cmds_addrs[0] = 0x50004;  in a660_build_bw_table()
462 	msg->ddr_cmds_addrs[1] = 0x500a0;  in a660_build_bw_table()
463 	msg->ddr_cmds_addrs[2] = 0x50000;  in a660_build_bw_table()
465 	msg->ddr_cmds_data[0][0] =  0x40000000;  in a660_build_bw_table()
466 	msg->ddr_cmds_data[0][1] =  0x40000000;  in a660_build_bw_table()
467 	msg->ddr_cmds_data[0][2] =  0x40000000;  in a660_build_bw_table()
474 	msg->cnoc_wait_bitmask = 0x01;  in a660_build_bw_table()
476 	msg->cnoc_cmds_addrs[0] = 0x50070;  in a660_build_bw_table()
477 	msg->cnoc_cmds_data[0][0] =  0x40000000;  in a660_build_bw_table()
478 	msg->cnoc_cmds_data[1][0] =  0x60000001;  in a660_build_bw_table()
490 	msg->ddr_wait_bitmask = 0x07;  in adreno_7c3_build_bw_table()
492 	msg->ddr_cmds_addrs[0] = 0x50004;  in adreno_7c3_build_bw_table()
493 	msg->ddr_cmds_addrs[1] = 0x50000;  in adreno_7c3_build_bw_table()
494 	msg->ddr_cmds_addrs[2] = 0x50088;  in adreno_7c3_build_bw_table()
496 	msg->ddr_cmds_data[0][0] =  0x40000000;  in adreno_7c3_build_bw_table()
497 	msg->ddr_cmds_data[0][1] =  0x40000000;  in adreno_7c3_build_bw_table()
498 	msg->ddr_cmds_data[0][2] =  0x40000000;  in adreno_7c3_build_bw_table()
505 	msg->cnoc_wait_bitmask = 0x01;  in adreno_7c3_build_bw_table()
507 	msg->cnoc_cmds_addrs[0] = 0x5006c;  in adreno_7c3_build_bw_table()
508 	msg->cnoc_cmds_data[0][0] =  0x40000000;  in adreno_7c3_build_bw_table()
509 	msg->cnoc_cmds_data[1][0] =  0x60000001;  in adreno_7c3_build_bw_table()
517 	msg->ddr_wait_bitmask = 0x7;  in a730_build_bw_table()
519 	msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0");  in a730_build_bw_table()
523 	msg->ddr_cmds_data[0][0] = 0x40000000;  in a730_build_bw_table()
524 	msg->ddr_cmds_data[0][1] = 0x40000000;  in a730_build_bw_table()
525 	msg->ddr_cmds_data[0][2] = 0x40000000;  in a730_build_bw_table()
526 	msg->ddr_cmds_data[1][0] = 0x600002e8;  in a730_build_bw_table()
527 	msg->ddr_cmds_data[1][1] = 0x600003d0;  in a730_build_bw_table()
528 	msg->ddr_cmds_data[1][2] = 0x60000008;  in a730_build_bw_table()
529 	msg->ddr_cmds_data[2][0] = 0x6000068d;  in a730_build_bw_table()
530 	msg->ddr_cmds_data[2][1] = 0x6000089a;  in a730_build_bw_table()
531 	msg->ddr_cmds_data[2][2] = 0x60000008;  in a730_build_bw_table()
532 	msg->ddr_cmds_data[3][0] = 0x600007f2;  in a730_build_bw_table()
533 	msg->ddr_cmds_data[3][1] = 0x60000a6e;  in a730_build_bw_table()
534 	msg->ddr_cmds_data[3][2] = 0x60000008;  in a730_build_bw_table()
535 	msg->ddr_cmds_data[4][0] = 0x600009e5;  in a730_build_bw_table()
536 	msg->ddr_cmds_data[4][1] = 0x60000cfd;  in a730_build_bw_table()
537 	msg->ddr_cmds_data[4][2] = 0x60000008;  in a730_build_bw_table()
538 	msg->ddr_cmds_data[5][0] = 0x60000b29;  in a730_build_bw_table()
539 	msg->ddr_cmds_data[5][1] = 0x60000ea6;  in a730_build_bw_table()
540 	msg->ddr_cmds_data[5][2] = 0x60000008;  in a730_build_bw_table()
541 	msg->ddr_cmds_data[6][0] = 0x60001698;  in a730_build_bw_table()
542 	msg->ddr_cmds_data[6][1] = 0x60001da8;  in a730_build_bw_table()
543 	msg->ddr_cmds_data[6][2] = 0x60000008;  in a730_build_bw_table()
544 	msg->ddr_cmds_data[7][0] = 0x600018d2;  in a730_build_bw_table()
545 	msg->ddr_cmds_data[7][1] = 0x60002093;  in a730_build_bw_table()
546 	msg->ddr_cmds_data[7][2] = 0x60000008;  in a730_build_bw_table()
547 	msg->ddr_cmds_data[8][0] = 0x60001e66;  in a730_build_bw_table()
548 	msg->ddr_cmds_data[8][1] = 0x600027e6;  in a730_build_bw_table()
549 	msg->ddr_cmds_data[8][2] = 0x60000008;  in a730_build_bw_table()
550 	msg->ddr_cmds_data[9][0] = 0x600027c2;  in a730_build_bw_table()
551 	msg->ddr_cmds_data[9][1] = 0x6000342f;  in a730_build_bw_table()
552 	msg->ddr_cmds_data[9][2] = 0x60000008;  in a730_build_bw_table()
553 	msg->ddr_cmds_data[10][0] = 0x60002e71;  in a730_build_bw_table()
554 	msg->ddr_cmds_data[10][1] = 0x60003cf5;  in a730_build_bw_table()
555 	msg->ddr_cmds_data[10][2] = 0x60000008;  in a730_build_bw_table()
556 	msg->ddr_cmds_data[11][0] = 0x600030ae;  in a730_build_bw_table()
557 	msg->ddr_cmds_data[11][1] = 0x60003fe5;  in a730_build_bw_table()
558 	msg->ddr_cmds_data[11][2] = 0x60000008;  in a730_build_bw_table()
561 	msg->cnoc_wait_bitmask = 0x1;  in a730_build_bw_table()
563 	msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");  in a730_build_bw_table()
564 	msg->cnoc_cmds_data[0][0] = 0x40000000;  in a730_build_bw_table()
565 	msg->cnoc_cmds_data[1][0] = 0x60000001;  in a730_build_bw_table()
573 	msg->ddr_wait_bitmask = 0x7;  in a740_build_bw_table()
575 	msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0");  in a740_build_bw_table()
579 	msg->ddr_cmds_data[0][0] = 0x40000000;  in a740_build_bw_table()
580 	msg->ddr_cmds_data[0][1] = 0x40000000;  in a740_build_bw_table()
581 	msg->ddr_cmds_data[0][2] = 0x40000000;  in a740_build_bw_table()
586 	msg->cnoc_wait_bitmask = 0x1;  in a740_build_bw_table()
588 	msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");  in a740_build_bw_table()
589 	msg->cnoc_cmds_data[0][0] = 0x40000000;  in a740_build_bw_table()
590 	msg->cnoc_cmds_data[1][0] = 0x60000001;  in a740_build_bw_table()
599 	msg->ddr_wait_bitmask = 0x07;  in a6xx_build_bw_table()
601 	msg->ddr_cmds_addrs[0] = 0x50000;  in a6xx_build_bw_table()
602 	msg->ddr_cmds_addrs[1] = 0x5005c;  in a6xx_build_bw_table()
603 	msg->ddr_cmds_addrs[2] = 0x5000c;  in a6xx_build_bw_table()
605 	msg->ddr_cmds_data[0][0] =  0x40000000;  in a6xx_build_bw_table()
606 	msg->ddr_cmds_data[0][1] =  0x40000000;  in a6xx_build_bw_table()
607 	msg->ddr_cmds_data[0][2] =  0x40000000;  in a6xx_build_bw_table()
615 	msg->cnoc_wait_bitmask = 0x05;  in a6xx_build_bw_table()
617 	msg->cnoc_cmds_addrs[0] = 0x50034;  in a6xx_build_bw_table()
618 	msg->cnoc_cmds_addrs[1] = 0x5007c;  in a6xx_build_bw_table()
619 	msg->cnoc_cmds_addrs[2] = 0x5004c;  in a6xx_build_bw_table()
621 	msg->cnoc_cmds_data[0][0] =  0x40000000;  in a6xx_build_bw_table()
622 	msg->cnoc_cmds_data[0][1] =  0x00000000;  in a6xx_build_bw_table()
623 	msg->cnoc_cmds_data[0][2] =  0x40000000;  in a6xx_build_bw_table()
625 	msg->cnoc_cmds_data[1][0] =  0x60000001;  in a6xx_build_bw_table()
626 	msg->cnoc_cmds_data[1][1] =  0x20000001;  in a6xx_build_bw_table()
627 	msg->cnoc_cmds_data[1][2] =  0x60000001;  in a6xx_build_bw_table()
633 	struct a6xx_hfi_msg_bw_table msg = { 0 };  in a6xx_hfi_send_bw_table()
659 		NULL, 0);  in a6xx_hfi_send_bw_table()
664 	struct a6xx_hfi_msg_test msg = { 0 };  in a6xx_hfi_send_test()
667 		NULL, 0);  in a6xx_hfi_send_test()
672 	struct a6xx_hfi_msg_start msg = { 0 };  in a6xx_hfi_send_start()
675 		NULL, 0);  in a6xx_hfi_send_start()
680 	struct a6xx_hfi_msg_core_fw_start msg = { 0 };  in a6xx_hfi_send_core_fw_start()
683 		sizeof(msg), NULL, 0);  in a6xx_hfi_send_core_fw_start()
688 	struct a6xx_hfi_gx_bw_perf_vote_cmd msg = { 0 };  in a6xx_hfi_set_freq()
692 	msg.bw = 0; /* TODO: bus scaling */  in a6xx_hfi_set_freq()
695 		sizeof(msg), NULL, 0);  in a6xx_hfi_set_freq()
700 	struct a6xx_hfi_prep_slumber_cmd msg = { 0 };  in a6xx_hfi_send_prep_slumber()
705 		sizeof(msg), NULL, 0);  in a6xx_hfi_send_prep_slumber()
740 	return 0;  in a6xx_hfi_start_v1()
771 	return 0;  in a6xx_hfi_start()
778 	for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {  in a6xx_hfi_stop()
787 		queue->header->read_index = 0;  in a6xx_hfi_stop()
788 		queue->header->write_index = 0;  in a6xx_hfi_stop()
790 		memset(&queue->history, 0xff, sizeof(queue->history));  in a6xx_hfi_stop()
791 		queue->history_idx = 0;  in a6xx_hfi_stop()
802 	atomic_set(&queue->seqnum, 0);  in a6xx_hfi_queue_init()
804 	memset(&queue->history, 0xff, sizeof(queue->history));  in a6xx_hfi_queue_init()
805 	queue->history_idx = 0;  in a6xx_hfi_queue_init()
812 	header->msg_size = 0;  in a6xx_hfi_queue_init()
813 	header->dropped = 0;  in a6xx_hfi_queue_init()
817 	header->tx_request = 0;  in a6xx_hfi_queue_init()
818 	header->read_index = 0;  in a6xx_hfi_queue_init()
819 	header->write_index = 0;  in a6xx_hfi_queue_init()
838 	table->version = 0;  in a6xx_hfi_init()
848 	a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset,  in a6xx_hfi_init()
849 		hfi->iova + offset, 0);  in a6xx_hfi_init()