Lines Matching full:device
15 #include <linux/device.h>
61 …U mode. This mode provides no device isolation, no DMA translation, no host kernel protection, ca…
66 int vfio_assign_device_set(struct vfio_device *device, void *set_id) in vfio_assign_device_set() argument
109 device->dev_set = dev_set; in vfio_assign_device_set()
110 list_add_tail(&device->dev_set_list, &dev_set->device_list); in vfio_assign_device_set()
116 static void vfio_release_device_set(struct vfio_device *device) in vfio_release_device_set() argument
118 struct vfio_device_set *dev_set = device->dev_set; in vfio_release_device_set()
124 list_del(&device->dev_set_list); in vfio_release_device_set()
152 struct device *dev) in vfio_find_device_in_devset()
166 * Device objects - create, release, get, put, search
168 /* Device reference always implies a group reference */
169 void vfio_device_put_registration(struct vfio_device *device) in vfio_device_put_registration() argument
171 if (refcount_dec_and_test(&device->refcount)) in vfio_device_put_registration()
172 complete(&device->comp); in vfio_device_put_registration()
175 bool vfio_device_try_get_registration(struct vfio_device *device) in vfio_device_try_get_registration() argument
177 return refcount_inc_not_zero(&device->refcount); in vfio_device_try_get_registration()
184 static void vfio_device_release(struct device *dev) in vfio_device_release()
186 struct vfio_device *device = in vfio_device_release() local
187 container_of(dev, struct vfio_device, device); in vfio_device_release()
189 vfio_release_device_set(device); in vfio_device_release()
190 ida_free(&vfio.device_ida, device->index); in vfio_device_release()
192 if (device->ops->release) in vfio_device_release()
193 device->ops->release(device); in vfio_device_release()
195 iput(device->inode); in vfio_device_release()
197 kvfree(device); in vfio_device_release()
200 static int vfio_init_device(struct vfio_device *device, struct device *dev,
211 * Driver may provide an @init callback to cover device private data.
215 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, in _vfio_alloc_device()
218 struct vfio_device *device; in _vfio_alloc_device() local
224 device = kvzalloc(size, GFP_KERNEL); in _vfio_alloc_device()
225 if (!device) in _vfio_alloc_device()
228 ret = vfio_init_device(device, dev, ops); in _vfio_alloc_device()
231 return device; in _vfio_alloc_device()
234 kvfree(device); in _vfio_alloc_device()
270 static int vfio_init_device(struct vfio_device *device, struct device *dev, in vfio_init_device() argument
281 device->index = ret; in vfio_init_device()
282 init_completion(&device->comp); in vfio_init_device()
283 device->dev = dev; in vfio_init_device()
284 device->ops = ops; in vfio_init_device()
285 device->inode = vfio_fs_inode_new(); in vfio_init_device()
286 if (IS_ERR(device->inode)) { in vfio_init_device()
287 ret = PTR_ERR(device->inode); in vfio_init_device()
292 ret = ops->init(device); in vfio_init_device()
297 device_initialize(&device->device); in vfio_init_device()
298 device->device.release = vfio_device_release; in vfio_init_device()
299 device->device.class = vfio.device_class; in vfio_init_device()
300 device->device.parent = device->dev; in vfio_init_device()
304 iput(device->inode); in vfio_init_device()
307 vfio_release_device_set(device); in vfio_init_device()
308 ida_free(&vfio.device_ida, device->index); in vfio_init_device()
312 static int __vfio_register_dev(struct vfio_device *device, in __vfio_register_dev() argument
318 (!device->ops->bind_iommufd || in __vfio_register_dev()
319 !device->ops->unbind_iommufd || in __vfio_register_dev()
320 !device->ops->attach_ioas || in __vfio_register_dev()
321 !device->ops->detach_ioas))) in __vfio_register_dev()
325 * If the driver doesn't specify a set then the device is added to a in __vfio_register_dev()
328 if (!device->dev_set) in __vfio_register_dev()
329 vfio_assign_device_set(device, device); in __vfio_register_dev()
331 ret = dev_set_name(&device->device, "vfio%d", device->index); in __vfio_register_dev()
335 ret = vfio_device_set_group(device, type); in __vfio_register_dev()
344 if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) && in __vfio_register_dev()
345 !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) { in __vfio_register_dev()
350 ret = vfio_device_add(device); in __vfio_register_dev()
355 refcount_set(&device->refcount, 1); in __vfio_register_dev()
357 vfio_device_group_register(device); in __vfio_register_dev()
358 vfio_device_debugfs_init(device); in __vfio_register_dev()
362 vfio_device_remove_group(device); in __vfio_register_dev()
366 int vfio_register_group_dev(struct vfio_device *device) in vfio_register_group_dev() argument
368 return __vfio_register_dev(device, VFIO_IOMMU); in vfio_register_group_dev()
373 * Register a virtual device without IOMMU backing. The user of this
374 * device must not be able to directly trigger unmediated DMA.
376 int vfio_register_emulated_iommu_dev(struct vfio_device *device) in vfio_register_emulated_iommu_dev() argument
378 return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); in vfio_register_emulated_iommu_dev()
383 * Decrement the device reference count and wait for the device to be
384 * removed. Open file descriptors for the device... */
385 void vfio_unregister_group_dev(struct vfio_device *device) in vfio_unregister_group_dev() argument
392 * Prevent new device opened by userspace via the in vfio_unregister_group_dev()
395 vfio_device_group_unregister(device); in vfio_unregister_group_dev()
399 * new device opened by userspace in the cdev path. in vfio_unregister_group_dev()
401 vfio_device_del(device); in vfio_unregister_group_dev()
403 vfio_device_put_registration(device); in vfio_unregister_group_dev()
404 rc = try_wait_for_completion(&device->comp); in vfio_unregister_group_dev()
406 if (device->ops->request) in vfio_unregister_group_dev()
407 device->ops->request(device, i++); in vfio_unregister_group_dev()
410 rc = wait_for_completion_timeout(&device->comp, in vfio_unregister_group_dev()
414 &device->comp, HZ * 10); in vfio_unregister_group_dev()
417 dev_warn(device->dev, in vfio_unregister_group_dev()
418 "Device is currently in use, task" in vfio_unregister_group_dev()
420 "blocked until device is released", in vfio_unregister_group_dev()
426 vfio_device_debugfs_exit(device); in vfio_unregister_group_dev()
428 vfio_device_remove_group(device); in vfio_unregister_group_dev()
433 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) in vfio_device_get_kvm_safe() argument
439 lockdep_assert_held(&device->dev_set->lock); in vfio_device_get_kvm_safe()
461 device->put_kvm = pfn; in vfio_device_get_kvm_safe()
462 device->kvm = kvm; in vfio_device_get_kvm_safe()
465 void vfio_device_put_kvm(struct vfio_device *device) in vfio_device_put_kvm() argument
467 lockdep_assert_held(&device->dev_set->lock); in vfio_device_put_kvm()
469 if (!device->kvm) in vfio_device_put_kvm()
472 if (WARN_ON(!device->put_kvm)) in vfio_device_put_kvm()
475 device->put_kvm(device->kvm); in vfio_device_put_kvm()
476 device->put_kvm = NULL; in vfio_device_put_kvm()
480 device->kvm = NULL; in vfio_device_put_kvm()
485 static bool vfio_assert_device_open(struct vfio_device *device) in vfio_assert_device_open() argument
487 return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); in vfio_assert_device_open()
491 vfio_allocate_device_file(struct vfio_device *device) in vfio_allocate_device_file() argument
499 df->device = device; in vfio_allocate_device_file()
507 struct vfio_device *device = df->device; in vfio_df_device_first_open() local
511 lockdep_assert_held(&device->dev_set->lock); in vfio_df_device_first_open()
513 if (!try_module_get(device->dev->driver->owner)) in vfio_df_device_first_open()
519 ret = vfio_device_group_use_iommu(device); in vfio_df_device_first_open()
523 if (device->ops->open_device) { in vfio_df_device_first_open()
524 ret = device->ops->open_device(device); in vfio_df_device_first_open()
534 vfio_device_group_unuse_iommu(device); in vfio_df_device_first_open()
536 module_put(device->dev->driver->owner); in vfio_df_device_first_open()
542 struct vfio_device *device = df->device; in vfio_df_device_last_close() local
545 lockdep_assert_held(&device->dev_set->lock); in vfio_df_device_last_close()
547 if (device->ops->close_device) in vfio_df_device_last_close()
548 device->ops->close_device(device); in vfio_df_device_last_close()
552 vfio_device_group_unuse_iommu(device); in vfio_df_device_last_close()
553 module_put(device->dev->driver->owner); in vfio_df_device_last_close()
558 struct vfio_device *device = df->device; in vfio_df_open() local
561 lockdep_assert_held(&device->dev_set->lock); in vfio_df_open()
564 * Only the group path allows the device to be opened multiple in vfio_df_open()
565 * times. The device cdev path doesn't have a secure way for it. in vfio_df_open()
567 if (device->open_count != 0 && !df->group) in vfio_df_open()
570 device->open_count++; in vfio_df_open()
571 if (device->open_count == 1) { in vfio_df_open()
574 device->open_count--; in vfio_df_open()
582 struct vfio_device *device = df->device; in vfio_df_close() local
584 lockdep_assert_held(&device->dev_set->lock); in vfio_df_close()
586 vfio_assert_device_open(device); in vfio_df_close()
587 if (device->open_count == 1) in vfio_df_close()
589 device->open_count--; in vfio_df_close()
596 static inline int vfio_device_pm_runtime_get(struct vfio_device *device) in vfio_device_pm_runtime_get() argument
598 struct device *dev = device->dev; in vfio_device_pm_runtime_get()
617 static inline void vfio_device_pm_runtime_put(struct vfio_device *device) in vfio_device_pm_runtime_put() argument
619 struct device *dev = device->dev; in vfio_device_pm_runtime_put()
626 * VFIO Device fd
631 struct vfio_device *device = df->device; in vfio_device_fops_release() local
638 vfio_device_put_registration(device); in vfio_device_fops_release()
647 * @cur_fsm - The current state the device is in
660 int vfio_mig_get_next_state(struct vfio_device *device, in vfio_mig_get_next_state() argument
832 (state_flags_table[cur_fsm] & device->migration_flags) != in vfio_mig_get_next_state()
837 (state_flags_table[new_fsm] & device->migration_flags) != in vfio_mig_get_next_state()
847 while ((state_flags_table[*next_fsm] & device->migration_flags) != in vfio_mig_get_next_state()
886 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, in vfio_ioctl_device_feature_mig_device_state() argument
896 if (!device->mig_ops) in vfio_ioctl_device_feature_mig_device_state()
912 ret = device->mig_ops->migration_get_state(device, in vfio_ioctl_device_feature_mig_device_state()
921 filp = device->mig_ops->migration_set_state(device, mig.device_state); in vfio_ioctl_device_feature_mig_device_state()
936 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, in vfio_ioctl_device_feature_migration_data_size() argument
944 if (!device->mig_ops) in vfio_ioctl_device_feature_migration_data_size()
952 ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); in vfio_ioctl_device_feature_migration_data_size()
963 static int vfio_ioctl_device_feature_migration(struct vfio_device *device, in vfio_ioctl_device_feature_migration() argument
968 .flags = device->migration_flags, in vfio_ioctl_device_feature_migration()
972 if (!device->mig_ops) in vfio_ioctl_device_feature_migration()
1046 vfio_ioctl_device_feature_logging_start(struct vfio_device *device, in vfio_ioctl_device_feature_logging_start() argument
1062 if (!device->log_ops) in vfio_ioctl_device_feature_logging_start()
1115 ret = device->log_ops->log_start(device, &root, nnodes, in vfio_ioctl_device_feature_logging_start()
1122 device->log_ops->log_stop(device); in vfio_ioctl_device_feature_logging_start()
1131 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, in vfio_ioctl_device_feature_logging_stop() argument
1137 if (!device->log_ops) in vfio_ioctl_device_feature_logging_stop()
1145 return device->log_ops->log_stop(device); in vfio_ioctl_device_feature_logging_stop()
1152 struct vfio_device *device = opaque; in vfio_device_log_read_and_clear() local
1154 return device->log_ops->log_read_and_clear(device, iova, length, iter); in vfio_device_log_read_and_clear()
1158 vfio_ioctl_device_feature_logging_report(struct vfio_device *device, in vfio_ioctl_device_feature_logging_report() argument
1170 if (!device->log_ops) in vfio_ioctl_device_feature_logging_report()
1195 ret = iova_bitmap_for_each(iter, device, in vfio_ioctl_device_feature_logging_report()
1202 static int vfio_ioctl_device_feature(struct vfio_device *device, in vfio_ioctl_device_feature() argument
1229 device, feature.flags, arg->data, in vfio_ioctl_device_feature()
1233 device, feature.flags, arg->data, in vfio_ioctl_device_feature()
1237 device, feature.flags, arg->data, in vfio_ioctl_device_feature()
1241 device, feature.flags, arg->data, in vfio_ioctl_device_feature()
1245 device, feature.flags, arg->data, in vfio_ioctl_device_feature()
1249 device, feature.flags, arg->data, in vfio_ioctl_device_feature()
1252 if (unlikely(!device->ops->device_feature)) in vfio_ioctl_device_feature()
1254 return device->ops->device_feature(device, feature.flags, in vfio_ioctl_device_feature()
1264 struct vfio_device *device = df->device; in vfio_device_fops_unl_ioctl() local
1275 ret = vfio_device_pm_runtime_get(device); in vfio_device_fops_unl_ioctl()
1294 ret = vfio_ioctl_device_feature(device, uptr); in vfio_device_fops_unl_ioctl()
1298 if (unlikely(!device->ops->ioctl)) in vfio_device_fops_unl_ioctl()
1301 ret = device->ops->ioctl(device, cmd, arg); in vfio_device_fops_unl_ioctl()
1305 vfio_device_pm_runtime_put(device); in vfio_device_fops_unl_ioctl()
1313 struct vfio_device *device = df->device; in vfio_device_fops_read() local
1319 if (unlikely(!device->ops->read)) in vfio_device_fops_read()
1322 return device->ops->read(device, buf, count, ppos); in vfio_device_fops_read()
1330 struct vfio_device *device = df->device; in vfio_device_fops_write() local
1336 if (unlikely(!device->ops->write)) in vfio_device_fops_write()
1339 return device->ops->write(device, buf, count, ppos); in vfio_device_fops_write()
1345 struct vfio_device *device = df->device; in vfio_device_fops_mmap() local
1351 if (unlikely(!device->ops->mmap)) in vfio_device_fops_mmap()
1354 return device->ops->mmap(device, vma); in vfio_device_fops_mmap()
1374 return df->device; in vfio_device_from_file()
1379 * @file: VFIO group file or VFIO device file
1391 * @file: VFIO group file or VFIO device file
1399 struct vfio_device *device; in vfio_file_enforced_coherent() local
1406 device = vfio_device_from_file(file); in vfio_file_enforced_coherent()
1407 if (device) in vfio_file_enforced_coherent()
1408 return device_iommu_capable(device->dev, in vfio_file_enforced_coherent()
1422 * iommufd successfully in the vfio device cdev path. in vfio_device_file_set_kvm()
1431 * @file: VFIO group file or VFIO device file
1434 * When a VFIO device is first opened the KVM will be available in
1435 * device->kvm if one was associated with the file.
1578 * @device [in] : device
1589 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, in vfio_pin_pages() argument
1592 /* group->container cannot change while a vfio device is open */ in vfio_pin_pages()
1593 if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) in vfio_pin_pages()
1595 if (!device->ops->dma_unmap) in vfio_pin_pages()
1597 if (vfio_device_has_container(device)) in vfio_pin_pages()
1598 return vfio_device_container_pin_pages(device, iova, in vfio_pin_pages()
1600 if (device->iommufd_access) { in vfio_pin_pages()
1612 device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), in vfio_pin_pages()
1625 * @device [in] : device
1630 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) in vfio_unpin_pages() argument
1632 if (WARN_ON(!vfio_assert_device_open(device))) in vfio_unpin_pages()
1634 if (WARN_ON(!device->ops->dma_unmap)) in vfio_unpin_pages()
1637 if (vfio_device_has_container(device)) { in vfio_unpin_pages()
1638 vfio_device_container_unpin_pages(device, iova, npage); in vfio_unpin_pages()
1641 if (device->iommufd_access) { in vfio_unpin_pages()
1644 iommufd_access_unpin_pages(device->iommufd_access, in vfio_unpin_pages()
1654 * behalf of the device.
1660 * not a real device DMA, it is not necessary to pin the user space memory.
1662 * @device [in] : VFIO device
1669 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, in vfio_dma_rw() argument
1672 if (!data || len <= 0 || !vfio_assert_device_open(device)) in vfio_dma_rw()
1675 if (vfio_device_has_container(device)) in vfio_dma_rw()
1676 return vfio_device_container_dma_rw(device, iova, in vfio_dma_rw()
1679 if (device->iommufd_access) { in vfio_dma_rw()
1690 return iommufd_access_rw(device->iommufd_access, iova, data, in vfio_dma_rw()