3  * Adjunct processor matrix VFIO device driver callbacks.
157 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&  in get_update_locks_by_apqn()
158 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {  in get_update_locks_by_apqn()
175  *			       KVM guest to which the matrix mdev linked to a
199  *			    hash table of queues assigned to a matrix mdev
200  * @matrix_mdev: the matrix mdev
634 				struct ap_matrix *matrix)  in vfio_ap_matrix_init()  argument
636 	matrix->apm_max = info->apxa ? info->na : 63;  in vfio_ap_matrix_init()
637 	matrix->aqm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
638 	matrix->adm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
655 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,  in vfio_ap_mdev_filter_cdoms()
678  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
683  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
689  *	 driver, its APID will be filtered from the guest's APCB. The matrix
713 	 * from the matrix mdev, but only those that are assigned to the host's  in vfio_ap_mdev_filter_matrix()
716 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_filter_matrix()
718 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_filter_matrix()
761 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);  in vfio_ap_mdev_init_dev()
827 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_fr_queues()
828 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_unlink_fr_queues()
871  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
877  * AP adapter IDs and AP queue indexes is not configured for any matrix
894 		if (mdev_apm == matrix_mdev->matrix.apm &&  in vfio_ap_mdev_verify_no_sharing()
895 		    mdev_aqm == matrix_mdev->matrix.aqm)  in vfio_ap_mdev_verify_no_sharing()
905 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_verify_no_sharing()
909 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_verify_no_sharing()
938 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
939 					       matrix_mdev->matrix.aqm))  in vfio_ap_mdev_validate_masks()
942 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
943 					      matrix_mdev->matrix.aqm);  in vfio_ap_mdev_validate_masks()
951 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)  in vfio_ap_mdev_link_adapter()
999  * corresponding bit in the mediated matrix device's APM
1001  * @dev:	the matrix device
1002  * @attr:	the mediated matrix device's assign_adapter attribute
1025  *	   matrix device
1047 	if (apid > matrix_mdev->matrix.apm_max) {  in assign_adapter_store()
1052 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in assign_adapter_store()
1057 	set_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1061 		clear_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1088 	/* If the queue is assigned to the matrix mdev, unlink it. */  in vfio_ap_unlink_apqn_fr_mdev()
1097  *				 adapter from the matrix mdev to which the
1099  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
1111 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in vfio_ap_mdev_unlink_adapter()
1165  * corresponding bit in the mediated matrix device's APM
1167  * @dev:	the matrix device
1168  * @attr:	the mediated matrix device's unassign_adapter attribute
1192 	if (apid > matrix_mdev->matrix.apm_max) {  in unassign_adapter_store()
1197 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in unassign_adapter_store()
1202 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);  in unassign_adapter_store()
1216 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)  in vfio_ap_mdev_link_domain()
1223  * corresponding bit in the mediated matrix device's AQM
1225  * @dev:	the matrix device
1226  * @attr:	the mediated matrix device's assign_domain attribute
1249  *	   matrix device
1271 	if (apqi > matrix_mdev->matrix.aqm_max) {  in assign_domain_store()
1276 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in assign_domain_store()
1281 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1285 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1312 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_domain()
1366  * corresponding bit in the mediated matrix device's AQM
1368  * @dev:	the matrix device
1369  * @attr:	the mediated matrix device's unassign_domain attribute
1393 	if (apqi > matrix_mdev->matrix.aqm_max) {  in unassign_domain_store()
1398 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in unassign_domain_store()
1403 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);  in unassign_domain_store()
1415  * the corresponding bit in the mediated matrix device's ADM
1417  * @dev:	the matrix device
1418  * @attr:	the mediated matrix device's assign_control_domain attribute
1441 	if (id > matrix_mdev->matrix.adm_max) {  in assign_control_domain_store()
1446 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {  in assign_control_domain_store()
1456 	set_bit_inv(id, matrix_mdev->matrix.adm);  in assign_control_domain_store()
1469  * clears the corresponding bit in the mediated matrix device's ADM
1471  * @dev:	the matrix device
1472  * @attr:	the mediated matrix device's unassign_control_domain attribute
1495 	if (domid > matrix_mdev->matrix.adm_max) {  in unassign_control_domain_store()
1500 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {  in unassign_control_domain_store()
1505 	clear_bit_inv(domid, matrix_mdev->matrix.adm);  in unassign_control_domain_store()
1528 	unsigned long max_domid = matrix_mdev->matrix.adm_max;  in control_domains_show()
1531 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {  in control_domains_show()
1542 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)  in vfio_ap_mdev_matrix_show()  argument
1549 	unsigned long napm_bits = matrix->apm_max + 1;  in vfio_ap_mdev_matrix_show()
1550 	unsigned long naqm_bits = matrix->aqm_max + 1;  in vfio_ap_mdev_matrix_show()
1554 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);  in vfio_ap_mdev_matrix_show()
1555 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);  in vfio_ap_mdev_matrix_show()
1558 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1559 			for_each_set_bit_inv(apqi, matrix->aqm,  in vfio_ap_mdev_matrix_show()
1568 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1574 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {  in vfio_ap_mdev_matrix_show()
1591 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);  in matrix_show()
1596 static DEVICE_ATTR_RO(matrix);
1624 	idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ',');  in ap_config_show()
1625 	idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ',');  in ap_config_show()
1626 	idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n');  in ap_config_show()
1652 	for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) {  in ap_matrix_overflow_check()
1653 		if (bit > matrix_mdev->matrix.apm_max)  in ap_matrix_overflow_check()
1657 	for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in ap_matrix_overflow_check()
1658 		if (bit > matrix_mdev->matrix.aqm_max)  in ap_matrix_overflow_check()
1662 	for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) {  in ap_matrix_overflow_check()
1663 		if (bit > matrix_mdev->matrix.adm_max)  in ap_matrix_overflow_check()
1701 	ap_matrix_copy(&m_old, &matrix_mdev->matrix);  in ap_config_store()
1715 	ap_matrix_copy(&matrix_mdev->matrix, &m_new);  in ap_config_store()
1720 		ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1725 		ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1731 	ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1738 	ap_matrix_copy(&matrix_mdev->matrix, &m_new);  in ap_config_store()
1791  * @matrix_mdev: a mediated matrix device
1794  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1855  * @matrix_mdev: a matrix mediated device
2229 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&  in vfio_ap_mdev_for_queue()
2230 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))  in vfio_ap_mdev_for_queue()
2251 	/* If the queue is assigned to the matrix mediated device, then  in status_show()
2261 		 * merely assigned to a matrix mediated device.  in status_show()
2377 		 * matrix and update the guest's AP configuration after all of  in vfio_ap_mdev_probe_queue()
2419 			 * Since the queues are defined via a matrix of adapters  in vfio_ap_mdev_remove_queue()
2544 					  matrix_mdev->matrix.apm,  in vfio_ap_mdev_cfg_remove()
2547 					  matrix_mdev->matrix.aqm,  in vfio_ap_mdev_cfg_remove()
2550 					     matrix_mdev->matrix.adm,  in vfio_ap_mdev_cfg_remove()
2662  *			  AP configuration for each matrix mdev to which they
2684 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2686 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);  in vfio_ap_mdev_cfg_add()
2688 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2758 	filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm,  in vfio_ap_mdev_hot_plug_cfg()
2760 	filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm,  in vfio_ap_mdev_hot_plug_cfg()
2762 	filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm,  in vfio_ap_mdev_hot_plug_cfg()