204 	int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
249 static inline bool perf_link_is_up(struct perf_peer *peer)  in perf_link_is_up()  argument
253 	link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);  in perf_link_is_up()
254 	return !!(link & BIT_ULL_MASK(peer->pidx));  in perf_link_is_up()
257 static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,  in perf_spad_cmd_send()  argument
260 	struct perf_ctx *perf = peer->perf;  in perf_spad_cmd_send()
274 		if (!perf_link_is_up(peer))  in perf_spad_cmd_send()
277 		sts = ntb_peer_spad_read(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
284 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
287 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
290 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
293 		ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));  in perf_spad_cmd_send()
296 			PERF_SPAD_NOTIFY(peer->gidx));  in perf_spad_cmd_send()
307 	struct perf_peer *peer;  in perf_spad_cmd_recv()  local
319 		peer = &perf->peers[*pidx];  in perf_spad_cmd_recv()
321 		if (!perf_link_is_up(peer))  in perf_spad_cmd_recv()
324 		val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));  in perf_spad_cmd_recv()
330 		val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));  in perf_spad_cmd_recv()
333 		val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));  in perf_spad_cmd_recv()
337 		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),  in perf_spad_cmd_recv()
348 static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,  in perf_msg_cmd_send()  argument
351 	struct perf_ctx *perf = peer->perf;  in perf_msg_cmd_send()
366 		if (!perf_link_is_up(peer))  in perf_msg_cmd_send()
373 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,  in perf_msg_cmd_send()
381 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,  in perf_msg_cmd_send()
385 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);  in perf_msg_cmd_send()
421 static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)  in perf_cmd_send()  argument
423 	struct perf_ctx *perf = peer->perf;  in perf_cmd_send()
426 		return perf->cmd_send(peer, cmd, data);  in perf_cmd_send()
432 static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)  in perf_cmd_exec()  argument
442 		dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");  in perf_cmd_exec()
447 	set_bit(cmd, &peer->sts);  in perf_cmd_exec()
449 	dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);  in perf_cmd_exec()
451 	(void)queue_work(system_highpri_wq, &peer->service);  in perf_cmd_exec()
458 	struct perf_peer *peer;  in perf_cmd_recv()  local
463 		peer = &perf->peers[pidx];  in perf_cmd_recv()
467 			peer->inbuf_size = data;  in perf_cmd_recv()
468 			return perf_cmd_exec(peer, PERF_CMD_RSIZE);  in perf_cmd_recv()
470 			peer->outbuf_xlat = data;  in perf_cmd_recv()
471 			return perf_cmd_exec(peer, PERF_CMD_RXLAT);  in perf_cmd_recv()
485 	struct perf_peer *peer;  in perf_link_event()  local
490 		peer = &perf->peers[pidx];  in perf_link_event()
492 		lnk_up = perf_link_is_up(peer);  in perf_link_event()
495 		    !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {  in perf_link_event()
496 			perf_cmd_exec(peer, PERF_CMD_SSIZE);  in perf_link_event()
498 			   test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {  in perf_link_event()
499 			perf_cmd_exec(peer, PERF_CMD_CLEAR);  in perf_link_event()
532 static void perf_free_outbuf(struct perf_peer *peer)  in perf_free_outbuf()  argument
534 	(void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_outbuf()
537 static int perf_setup_outbuf(struct perf_peer *peer)  in perf_setup_outbuf()  argument
539 	struct perf_ctx *perf = peer->perf;  in perf_setup_outbuf()
543 	ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_outbuf()
544 				    peer->outbuf_xlat, peer->outbuf_size);  in perf_setup_outbuf()
551 	set_bit(PERF_STS_DONE, &peer->sts);  in perf_setup_outbuf()
552 	complete_all(&peer->init_comp);  in perf_setup_outbuf()
557 static void perf_free_inbuf(struct perf_peer *peer)  in perf_free_inbuf()  argument
559 	if (!peer->inbuf)  in perf_free_inbuf()
562 	(void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_inbuf()
563 	dma_free_coherent(&peer->perf->ntb->pdev->dev, peer->inbuf_size,  in perf_free_inbuf()
564 			  peer->inbuf, peer->inbuf_xlat);  in perf_free_inbuf()
565 	peer->inbuf = NULL;  in perf_free_inbuf()
568 static int perf_setup_inbuf(struct perf_peer *peer)  in perf_setup_inbuf()  argument
571 	struct perf_ctx *perf = peer->perf;  in perf_setup_inbuf()
575 	ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,  in perf_setup_inbuf()
582 	if (peer->inbuf_size > size_max) {  in perf_setup_inbuf()
584 			&peer->inbuf_size, &size_max);  in perf_setup_inbuf()
588 	peer->inbuf_size = round_up(peer->inbuf_size, size_align);  in perf_setup_inbuf()
590 	perf_free_inbuf(peer);  in perf_setup_inbuf()
592 	peer->inbuf = dma_alloc_coherent(&perf->ntb->pdev->dev,  in perf_setup_inbuf()
593 					 peer->inbuf_size, &peer->inbuf_xlat,  in perf_setup_inbuf()
595 	if (!peer->inbuf) {  in perf_setup_inbuf()
597 			&peer->inbuf_size);  in perf_setup_inbuf()
600 	if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {  in perf_setup_inbuf()
606 	ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_inbuf()
607 			       peer->inbuf_xlat, peer->inbuf_size);  in perf_setup_inbuf()
618 	(void)perf_cmd_exec(peer, PERF_CMD_SXLAT);  in perf_setup_inbuf()
623 	perf_free_inbuf(peer);  in perf_setup_inbuf()
630 	struct perf_peer *peer = to_peer_service(work);  in perf_service_work()  local
632 	if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))  in perf_service_work()
633 		perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);  in perf_service_work()
635 	if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))  in perf_service_work()
636 		perf_setup_inbuf(peer);  in perf_service_work()
638 	if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))  in perf_service_work()
639 		perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);  in perf_service_work()
641 	if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))  in perf_service_work()
642 		perf_setup_outbuf(peer);  in perf_service_work()
644 	if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {  in perf_service_work()
645 		init_completion(&peer->init_comp);  in perf_service_work()
646 		clear_bit(PERF_STS_DONE, &peer->sts);  in perf_service_work()
647 		if (test_bit(0, &peer->perf->busy_flag) &&  in perf_service_work()
648 		    peer == peer->perf->test_peer) {  in perf_service_work()
649 			dev_warn(&peer->perf->ntb->dev,  in perf_service_work()
651 			perf_terminate_test(peer->perf);  in perf_service_work()
653 		perf_free_outbuf(peer);  in perf_service_work()
654 		perf_free_inbuf(peer);  in perf_service_work()
762 		struct perf_peer *peer = &perf->peers[pidx];  in perf_disable_service()  local
764 		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx), 0);  in perf_disable_service()
792 	struct perf_peer *peer = pthr->perf->test_peer;  in perf_copy_chunk()  local
808 	vbase = peer->outbuf;  in perf_copy_chunk()
810 	dst_dma_addr = peer->dma_dst_addr + (dst_vaddr - vbase);  in perf_copy_chunk()
875 	struct perf_peer *peer = pthr->perf->test_peer;  in perf_init_test()  local
895 	peer->dma_dst_addr =  in perf_init_test()
897 				 peer->out_phys_addr, peer->outbuf_size,  in perf_init_test()
900 			      peer->dma_dst_addr)) {  in perf_init_test()
903 		peer->dma_dst_addr = 0;  in perf_init_test()
909 			&peer->out_phys_addr,  in perf_init_test()
910 			&peer->dma_dst_addr);  in perf_init_test()
924 	struct perf_peer *peer = pthr->perf->test_peer;  in perf_run_test()  local
933 	chunk_size = min_t(u64, peer->outbuf_size, chunk_size);  in perf_run_test()
936 	bnd_dst = peer->outbuf + peer->outbuf_size;  in perf_run_test()
937 	flt_dst = peer->outbuf;  in perf_run_test()
954 		if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {  in perf_run_test()
955 			flt_dst = peer->outbuf;  in perf_run_test()
1079 static int perf_submit_test(struct perf_peer *peer)  in perf_submit_test()  argument
1081 	struct perf_ctx *perf = peer->perf;  in perf_submit_test()
1085 	ret = wait_for_completion_interruptible(&peer->init_comp);  in perf_submit_test()
1092 	perf->test_peer = peer;  in perf_submit_test()
1186 	struct perf_peer *peer;  in perf_dbgfs_read_info()  local
1215 		peer = &perf->peers[pidx];  in perf_dbgfs_read_info()
1219 			ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,  in perf_dbgfs_read_info()
1220 			peer->gidx);  in perf_dbgfs_read_info()
1224 			test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");  in perf_dbgfs_read_info()
1227 			"\tOut buffer addr 0x%pK\n", peer->outbuf);  in perf_dbgfs_read_info()
1230 			"\tOut buff phys addr %pap\n", &peer->out_phys_addr);  in perf_dbgfs_read_info()
1233 			"\tOut buffer size %pa\n", &peer->outbuf_size);  in perf_dbgfs_read_info()
1236 			"\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);  in perf_dbgfs_read_info()
1238 		if (!peer->inbuf) {  in perf_dbgfs_read_info()
1245 			"\tIn buffer addr 0x%pK\n", peer->inbuf);  in perf_dbgfs_read_info()
1248 			"\tIn buffer size %pa\n", &peer->inbuf_size);  in perf_dbgfs_read_info()
1251 			"\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);  in perf_dbgfs_read_info()
1291 	struct perf_peer *peer;  in perf_dbgfs_write_run()  local
1301 	peer = &perf->peers[pidx];  in perf_dbgfs_write_run()
1303 	ret = perf_submit_test(peer);  in perf_dbgfs_write_run()
1409 static int perf_setup_peer_mw(struct perf_peer *peer)  in perf_setup_peer_mw()  argument
1411 	struct perf_ctx *perf = peer->perf;  in perf_setup_peer_mw()
1417 				   &peer->outbuf_size);  in perf_setup_peer_mw()
1421 	peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,  in perf_setup_peer_mw()
1422 					peer->outbuf_size);  in perf_setup_peer_mw()
1423 	if (!peer->outbuf)  in perf_setup_peer_mw()
1426 	peer->out_phys_addr = phys_addr;  in perf_setup_peer_mw()
1428 	if (max_mw_size && peer->outbuf_size > max_mw_size) {  in perf_setup_peer_mw()
1429 		peer->outbuf_size = max_mw_size;  in perf_setup_peer_mw()
1430 		dev_warn(&peer->perf->ntb->dev,  in perf_setup_peer_mw()
1431 			"Peer %d outbuf reduced to %pa\n", peer->pidx,  in perf_setup_peer_mw()
1432 			&peer->outbuf_size);  in perf_setup_peer_mw()
1440 	struct perf_peer *peer;  in perf_init_peers()  local
1446 		peer = &perf->peers[pidx];  in perf_init_peers()
1448 		peer->perf = perf;  in perf_init_peers()
1449 		peer->pidx = pidx;  in perf_init_peers()
1453 			peer->gidx = pidx + 1;  in perf_init_peers()
1455 			peer->gidx = pidx;  in perf_init_peers()
1457 		INIT_WORK(&peer->service, perf_service_work);  in perf_init_peers()
1458 		init_completion(&peer->init_comp);  in perf_init_peers()