82 	if (!hwpt->fault)  in iommufd_fault_domain_attach_dev()
99 	struct iommufd_fault *fault = hwpt->fault;  in iommufd_auto_response_faults()  local
103 	if (!fault)  in iommufd_auto_response_faults()
106 	mutex_lock(&fault->mutex);  in iommufd_auto_response_faults()
107 	list_for_each_entry_safe(group, next, &fault->deliver, node) {  in iommufd_auto_response_faults()
115 	xa_for_each(&fault->response, index, group) {  in iommufd_auto_response_faults()
118 		xa_erase(&fault->response, index);  in iommufd_auto_response_faults()
122 	mutex_unlock(&fault->mutex);  in iommufd_auto_response_faults()
156 	if (old->fault)  in __fault_domain_replace_dev()
159 	if (hwpt->fault) {  in __fault_domain_replace_dev()
184 	bool iopf_off = !hwpt->fault && old->fault;  in iommufd_fault_domain_replace_dev()
185 	bool iopf_on = hwpt->fault && !old->fault;  in iommufd_fault_domain_replace_dev()
209 	struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);  in iommufd_fault_destroy()  local
218 	list_for_each_entry_safe(group, next, &fault->deliver, node) {  in iommufd_fault_destroy()
225 static void iommufd_compose_fault_message(struct iommu_fault *fault,  in iommufd_compose_fault_message()  argument
230 	hwpt_fault->flags = fault->prm.flags;  in iommufd_compose_fault_message()
232 	hwpt_fault->pasid = fault->prm.pasid;  in iommufd_compose_fault_message()
233 	hwpt_fault->grpid = fault->prm.grpid;  in iommufd_compose_fault_message()
234 	hwpt_fault->perm = fault->prm.perm;  in iommufd_compose_fault_message()
235 	hwpt_fault->addr = fault->prm.addr;  in iommufd_compose_fault_message()
244 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_read()  local
255 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_read()
256 	while (!list_empty(&fault->deliver) && count > done) {  in iommufd_fault_fops_read()
257 		group = list_first_entry(&fault->deliver,  in iommufd_fault_fops_read()
263 		rc = xa_alloc(&fault->response, &group->cookie, group,  in iommufd_fault_fops_read()
270 			iommufd_compose_fault_message(&iopf->fault,  in iommufd_fault_fops_read()
274 				xa_erase(&fault->response, group->cookie);  in iommufd_fault_fops_read()
283 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_read()
292 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_write()  local
301 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_write()
317 		group = xa_erase(&fault->response, response.cookie);  in iommufd_fault_fops_write()
327 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_write()
335 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_poll()  local
338 	poll_wait(filep, &fault->wait_queue, wait);  in iommufd_fault_fops_poll()
339 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_poll()
340 	if (!list_empty(&fault->deliver))  in iommufd_fault_fops_poll()
342 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_poll()
349 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_release()  local
351 	refcount_dec(&fault->obj.users);  in iommufd_fault_fops_release()
352 	iommufd_ctx_put(fault->ictx);  in iommufd_fault_fops_release()
368 	struct iommufd_fault *fault;  in iommufd_fault_alloc()  local
376 	fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);  in iommufd_fault_alloc()
377 	if (IS_ERR(fault))  in iommufd_fault_alloc()
378 		return PTR_ERR(fault);  in iommufd_fault_alloc()
380 	fault->ictx = ucmd->ictx;  in iommufd_fault_alloc()
381 	INIT_LIST_HEAD(&fault->deliver);  in iommufd_fault_alloc()
382 	xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);  in iommufd_fault_alloc()
383 	mutex_init(&fault->mutex);  in iommufd_fault_alloc()
384 	init_waitqueue_head(&fault->wait_queue);  in iommufd_fault_alloc()
387 				   fault, O_RDWR);  in iommufd_fault_alloc()
393 	refcount_inc(&fault->obj.users);  in iommufd_fault_alloc()
394 	iommufd_ctx_get(fault->ictx);  in iommufd_fault_alloc()
395 	fault->filep = filep;  in iommufd_fault_alloc()
403 	cmd->out_fault_id = fault->obj.id;  in iommufd_fault_alloc()
409 	iommufd_object_finalize(ucmd->ictx, &fault->obj);  in iommufd_fault_alloc()
411 	fd_install(fdno, fault->filep);  in iommufd_fault_alloc()
418 	refcount_dec(&fault->obj.users);  in iommufd_fault_alloc()
419 	iommufd_ctx_put(fault->ictx);  in iommufd_fault_alloc()
421 	iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);  in iommufd_fault_alloc()
429 	struct iommufd_fault *fault;  in iommufd_fault_iopf_handler()  local
432 	fault = hwpt->fault;  in iommufd_fault_iopf_handler()
434 	mutex_lock(&fault->mutex);  in iommufd_fault_iopf_handler()
435 	list_add_tail(&group->node, &fault->deliver);  in iommufd_fault_iopf_handler()
436 	mutex_unlock(&fault->mutex);  in iommufd_fault_iopf_handler()
438 	wake_up_interruptible(&fault->wait_queue);  in iommufd_fault_iopf_handler()