Lines Matching full:vdev
108 static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev) in vfio_pci_probe_mmaps() argument
117 res = &vdev->pdev->resource[bar]; in vfio_pci_probe_mmaps()
134 vdev->bar_mmap_supported[bar] = true; in vfio_pci_probe_mmaps()
160 &vdev->dummy_resources_list); in vfio_pci_probe_mmaps()
161 vdev->bar_mmap_supported[bar] = true; in vfio_pci_probe_mmaps()
173 vdev->bar_mmap_supported[bar] = false; in vfio_pci_probe_mmaps()
214 static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev) in vfio_pci_probe_power_state() argument
216 struct pci_dev *pdev = vdev->pdev; in vfio_pci_probe_power_state()
224 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); in vfio_pci_probe_power_state()
229 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,
234 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state) in vfio_pci_set_power_state() argument
236 struct pci_dev *pdev = vdev->pdev; in vfio_pci_set_power_state()
244 if (vdev->needs_pm_restore) { in vfio_pci_set_power_state()
271 kfree(vdev->pm_save); in vfio_pci_set_power_state()
272 vdev->pm_save = pci_store_saved_state(pdev); in vfio_pci_set_power_state()
274 pci_load_and_free_saved_state(pdev, &vdev->pm_save); in vfio_pci_set_power_state()
282 static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev, in vfio_pci_runtime_pm_entry() argument
286 * The vdev power related flags are protected with 'memory_lock' in vfio_pci_runtime_pm_entry()
289 vfio_pci_zap_and_down_write_memory_lock(vdev); in vfio_pci_runtime_pm_entry()
290 if (vdev->pm_runtime_engaged) { in vfio_pci_runtime_pm_entry()
291 up_write(&vdev->memory_lock); in vfio_pci_runtime_pm_entry()
295 vdev->pm_runtime_engaged = true; in vfio_pci_runtime_pm_entry()
296 vdev->pm_wake_eventfd_ctx = efdctx; in vfio_pci_runtime_pm_entry()
297 pm_runtime_put_noidle(&vdev->pdev->dev); in vfio_pci_runtime_pm_entry()
298 up_write(&vdev->memory_lock); in vfio_pci_runtime_pm_entry()
306 struct vfio_pci_core_device *vdev = in vfio_pci_core_pm_entry() local
307 container_of(device, struct vfio_pci_core_device, vdev); in vfio_pci_core_pm_entry()
320 return vfio_pci_runtime_pm_entry(vdev, NULL); in vfio_pci_core_pm_entry()
328 struct vfio_pci_core_device *vdev = in vfio_pci_core_pm_entry_with_wakeup() local
329 container_of(device, struct vfio_pci_core_device, vdev); in vfio_pci_core_pm_entry_with_wakeup()
349 ret = vfio_pci_runtime_pm_entry(vdev, efdctx); in vfio_pci_core_pm_entry_with_wakeup()
356 static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev) in __vfio_pci_runtime_pm_exit() argument
358 if (vdev->pm_runtime_engaged) { in __vfio_pci_runtime_pm_exit()
359 vdev->pm_runtime_engaged = false; in __vfio_pci_runtime_pm_exit()
360 pm_runtime_get_noresume(&vdev->pdev->dev); in __vfio_pci_runtime_pm_exit()
362 if (vdev->pm_wake_eventfd_ctx) { in __vfio_pci_runtime_pm_exit()
363 eventfd_ctx_put(vdev->pm_wake_eventfd_ctx); in __vfio_pci_runtime_pm_exit()
364 vdev->pm_wake_eventfd_ctx = NULL; in __vfio_pci_runtime_pm_exit()
369 static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev) in vfio_pci_runtime_pm_exit() argument
372 * The vdev power related flags are protected with 'memory_lock' in vfio_pci_runtime_pm_exit()
375 down_write(&vdev->memory_lock); in vfio_pci_runtime_pm_exit()
376 __vfio_pci_runtime_pm_exit(vdev); in vfio_pci_runtime_pm_exit()
377 up_write(&vdev->memory_lock); in vfio_pci_runtime_pm_exit()
383 struct vfio_pci_core_device *vdev = in vfio_pci_core_pm_exit() local
384 container_of(device, struct vfio_pci_core_device, vdev); in vfio_pci_core_pm_exit()
398 vfio_pci_runtime_pm_exit(vdev); in vfio_pci_core_pm_exit()
405 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); in vfio_pci_core_runtime_suspend() local
407 down_write(&vdev->memory_lock); in vfio_pci_core_runtime_suspend()
414 * (saved locally in 'vdev->pm_save'). in vfio_pci_core_runtime_suspend()
416 vfio_pci_set_power_state(vdev, PCI_D0); in vfio_pci_core_runtime_suspend()
417 up_write(&vdev->memory_lock); in vfio_pci_core_runtime_suspend()
426 vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) && in vfio_pci_core_runtime_suspend()
427 vfio_pci_intx_mask(vdev)); in vfio_pci_core_runtime_suspend()
434 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); in vfio_pci_core_runtime_resume() local
440 down_write(&vdev->memory_lock); in vfio_pci_core_runtime_resume()
441 if (vdev->pm_wake_eventfd_ctx) { in vfio_pci_core_runtime_resume()
442 eventfd_signal(vdev->pm_wake_eventfd_ctx); in vfio_pci_core_runtime_resume()
443 __vfio_pci_runtime_pm_exit(vdev); in vfio_pci_core_runtime_resume()
445 up_write(&vdev->memory_lock); in vfio_pci_core_runtime_resume()
447 if (vdev->pm_intx_masked) in vfio_pci_core_runtime_resume()
448 vfio_pci_intx_unmask(vdev); in vfio_pci_core_runtime_resume()
466 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) in vfio_pci_core_enable() argument
468 struct pci_dev *pdev = vdev->pdev; in vfio_pci_core_enable()
491 vdev->reset_works = !ret; in vfio_pci_core_enable()
493 vdev->pci_saved_state = pci_store_saved_state(pdev); in vfio_pci_core_enable()
494 if (!vdev->pci_saved_state) in vfio_pci_core_enable()
500 vdev->nointx = true; in vfio_pci_core_enable()
503 vdev->pci_2_3 = pci_intx_mask_supported(pdev); in vfio_pci_core_enable()
507 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { in vfio_pci_core_enable()
512 ret = vfio_pci_zdev_open_device(vdev); in vfio_pci_core_enable()
516 ret = vfio_config_init(vdev); in vfio_pci_core_enable()
528 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; in vfio_pci_core_enable()
529 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; in vfio_pci_core_enable()
530 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; in vfio_pci_core_enable()
531 vdev->has_dyn_msix = pci_msix_can_alloc_dyn(pdev); in vfio_pci_core_enable()
533 vdev->msix_bar = 0xFF; in vfio_pci_core_enable()
534 vdev->has_dyn_msix = false; in vfio_pci_core_enable()
538 vdev->has_vga = true; in vfio_pci_core_enable()
544 vfio_pci_zdev_close_device(vdev); in vfio_pci_core_enable()
546 kfree(vdev->pci_saved_state); in vfio_pci_core_enable()
547 vdev->pci_saved_state = NULL; in vfio_pci_core_enable()
557 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) in vfio_pci_core_disable() argument
559 struct pci_dev *pdev = vdev->pdev; in vfio_pci_core_disable()
565 lockdep_assert_held(&vdev->vdev.dev_set->lock); in vfio_pci_core_disable()
577 vfio_pci_runtime_pm_exit(vdev); in vfio_pci_core_disable()
586 * 'vdev->pm_save'). in vfio_pci_core_disable()
588 vfio_pci_set_power_state(vdev, PCI_D0); in vfio_pci_core_disable()
593 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | in vfio_pci_core_disable()
595 vdev->irq_type, 0, 0, NULL); in vfio_pci_core_disable()
599 &vdev->ioeventfds_list, next) { in vfio_pci_core_disable()
604 vdev->ioeventfds_nr = 0; in vfio_pci_core_disable()
606 vdev->virq_disabled = false; in vfio_pci_core_disable()
608 for (i = 0; i < vdev->num_regions; i++) in vfio_pci_core_disable()
609 vdev->region[i].ops->release(vdev, &vdev->region[i]); in vfio_pci_core_disable()
611 vdev->num_regions = 0; in vfio_pci_core_disable()
612 kfree(vdev->region); in vfio_pci_core_disable()
613 vdev->region = NULL; /* don't krealloc a freed pointer */ in vfio_pci_core_disable()
615 vfio_config_free(vdev); in vfio_pci_core_disable()
619 if (!vdev->barmap[bar]) in vfio_pci_core_disable()
621 pci_iounmap(pdev, vdev->barmap[bar]); in vfio_pci_core_disable()
623 vdev->barmap[bar] = NULL; in vfio_pci_core_disable()
627 &vdev->dummy_resources_list, res_next) { in vfio_pci_core_disable()
633 vdev->needs_reset = true; in vfio_pci_core_disable()
635 vfio_pci_zdev_close_device(vdev); in vfio_pci_core_disable()
643 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { in vfio_pci_core_disable()
646 if (!vdev->reset_works) in vfio_pci_core_disable()
665 if (vdev->reset_works && pci_dev_trylock(pdev)) { in vfio_pci_core_disable()
667 vdev->needs_reset = false; in vfio_pci_core_disable()
675 vfio_pci_dev_set_try_reset(vdev->vdev.dev_set); in vfio_pci_core_disable()
685 struct vfio_pci_core_device *vdev = in vfio_pci_core_close_device() local
686 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_close_device()
688 if (vdev->sriov_pf_core_dev) { in vfio_pci_core_close_device()
689 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock); in vfio_pci_core_close_device()
690 WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users); in vfio_pci_core_close_device()
691 vdev->sriov_pf_core_dev->vf_token->users--; in vfio_pci_core_close_device()
692 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock); in vfio_pci_core_close_device()
695 eeh_dev_release(vdev->pdev); in vfio_pci_core_close_device()
697 vfio_pci_core_disable(vdev); in vfio_pci_core_close_device()
699 mutex_lock(&vdev->igate); in vfio_pci_core_close_device()
700 if (vdev->err_trigger) { in vfio_pci_core_close_device()
701 eventfd_ctx_put(vdev->err_trigger); in vfio_pci_core_close_device()
702 vdev->err_trigger = NULL; in vfio_pci_core_close_device()
704 if (vdev->req_trigger) { in vfio_pci_core_close_device()
705 eventfd_ctx_put(vdev->req_trigger); in vfio_pci_core_close_device()
706 vdev->req_trigger = NULL; in vfio_pci_core_close_device()
708 mutex_unlock(&vdev->igate); in vfio_pci_core_close_device()
712 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev) in vfio_pci_core_finish_enable() argument
714 vfio_pci_probe_mmaps(vdev); in vfio_pci_core_finish_enable()
716 eeh_dev_open(vdev->pdev); in vfio_pci_core_finish_enable()
719 if (vdev->sriov_pf_core_dev) { in vfio_pci_core_finish_enable()
720 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock); in vfio_pci_core_finish_enable()
721 vdev->sriov_pf_core_dev->vf_token->users++; in vfio_pci_core_finish_enable()
722 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock); in vfio_pci_core_finish_enable()
727 static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type) in vfio_pci_get_irq_count() argument
733 vdev->nointx || vdev->pdev->is_virtfn) in vfio_pci_get_irq_count()
736 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); in vfio_pci_get_irq_count()
743 pos = vdev->pdev->msi_cap; in vfio_pci_get_irq_count()
745 pci_read_config_word(vdev->pdev, in vfio_pci_get_irq_count()
753 pos = vdev->pdev->msix_cap; in vfio_pci_get_irq_count()
755 pci_read_config_word(vdev->pdev, in vfio_pci_get_irq_count()
761 if (pci_is_pcie(vdev->pdev)) in vfio_pci_get_irq_count()
777 struct vfio_device *vdev; member
799 struct iommufd_ctx *iommufd = vfio_iommufd_device_ictx(fill->vdev); in vfio_pci_fill_devs()
800 struct vfio_device_set *dev_set = fill->vdev->dev_set; in vfio_pci_fill_devs()
801 struct vfio_device *vdev; in vfio_pci_fill_devs() local
807 vdev = vfio_find_device_in_devset(dev_set, &pdev->dev); in vfio_pci_fill_devs()
808 if (!vdev) { in vfio_pci_fill_devs()
811 int id = vfio_iommufd_get_dev_id(vdev, iommufd); in vfio_pci_fill_devs()
882 static int msix_mmappable_cap(struct vfio_pci_core_device *vdev, in msix_mmappable_cap() argument
893 int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, in vfio_pci_core_register_dev_region() argument
900 region = krealloc(vdev->region, in vfio_pci_core_register_dev_region()
901 (vdev->num_regions + 1) * sizeof(*region), in vfio_pci_core_register_dev_region()
906 vdev->region = region; in vfio_pci_core_register_dev_region()
907 vdev->region[vdev->num_regions].type = type; in vfio_pci_core_register_dev_region()
908 vdev->region[vdev->num_regions].subtype = subtype; in vfio_pci_core_register_dev_region()
909 vdev->region[vdev->num_regions].ops = ops; in vfio_pci_core_register_dev_region()
910 vdev->region[vdev->num_regions].size = size; in vfio_pci_core_register_dev_region()
911 vdev->region[vdev->num_regions].flags = flags; in vfio_pci_core_register_dev_region()
912 vdev->region[vdev->num_regions].data = data; in vfio_pci_core_register_dev_region()
914 vdev->num_regions++; in vfio_pci_core_register_dev_region()
920 static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev, in vfio_pci_info_atomic_cap() argument
927 struct pci_dev *pdev = pci_physfn(vdev->pdev); in vfio_pci_info_atomic_cap()
951 static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_get_info() argument
969 if (vdev->reset_works) in vfio_pci_ioctl_get_info()
972 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; in vfio_pci_ioctl_get_info()
975 ret = vfio_pci_info_zdev_add_caps(vdev, &caps); in vfio_pci_ioctl_get_info()
977 pci_warn(vdev->pdev, in vfio_pci_ioctl_get_info()
982 ret = vfio_pci_info_atomic_cap(vdev, &caps); in vfio_pci_ioctl_get_info()
984 pci_warn(vdev->pdev, in vfio_pci_ioctl_get_info()
1008 static int vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_get_region_info() argument
1012 struct pci_dev *pdev = vdev->pdev; in vfio_pci_ioctl_get_region_info()
1040 if (vdev->bar_mmap_supported[info.index]) { in vfio_pci_ioctl_get_region_info()
1042 if (info.index == vdev->msix_bar) { in vfio_pci_ioctl_get_region_info()
1043 ret = msix_mmappable_cap(vdev, &caps); in vfio_pci_ioctl_get_region_info()
1073 cmd = vfio_pci_memory_lock_and_enable(vdev); in vfio_pci_ioctl_get_region_info()
1081 vfio_pci_memory_unlock_and_restore(vdev, cmd); in vfio_pci_ioctl_get_region_info()
1086 if (!vdev->has_vga) in vfio_pci_ioctl_get_region_info()
1101 if (info.index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) in vfio_pci_ioctl_get_region_info()
1104 info.index, VFIO_PCI_NUM_REGIONS + vdev->num_regions); in vfio_pci_ioctl_get_region_info()
1109 info.size = vdev->region[i].size; in vfio_pci_ioctl_get_region_info()
1110 info.flags = vdev->region[i].flags; in vfio_pci_ioctl_get_region_info()
1112 cap_type.type = vdev->region[i].type; in vfio_pci_ioctl_get_region_info()
1113 cap_type.subtype = vdev->region[i].subtype; in vfio_pci_ioctl_get_region_info()
1120 if (vdev->region[i].ops->add_capability) { in vfio_pci_ioctl_get_region_info()
1121 ret = vdev->region[i].ops->add_capability( in vfio_pci_ioctl_get_region_info()
1122 vdev, &vdev->region[i], &caps); in vfio_pci_ioctl_get_region_info()
1149 static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_get_irq_info() argument
1166 if (pci_is_pcie(vdev->pdev)) in vfio_pci_ioctl_get_irq_info()
1175 info.count = vfio_pci_get_irq_count(vdev, info.index); in vfio_pci_ioctl_get_irq_info()
1180 else if (info.index != VFIO_PCI_MSIX_IRQ_INDEX || !vdev->has_dyn_msix) in vfio_pci_ioctl_get_irq_info()
1186 static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_set_irqs() argument
1198 max = vfio_pci_get_irq_count(vdev, hdr.index); in vfio_pci_ioctl_set_irqs()
1211 mutex_lock(&vdev->igate); in vfio_pci_ioctl_set_irqs()
1213 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start, in vfio_pci_ioctl_set_irqs()
1216 mutex_unlock(&vdev->igate); in vfio_pci_ioctl_set_irqs()
1222 static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_reset() argument
1227 if (!vdev->reset_works) in vfio_pci_ioctl_reset()
1230 vfio_pci_zap_and_down_write_memory_lock(vdev); in vfio_pci_ioctl_reset()
1239 * 'vdev->pm_save'). in vfio_pci_ioctl_reset()
1241 vfio_pci_set_power_state(vdev, PCI_D0); in vfio_pci_ioctl_reset()
1243 ret = pci_try_reset_function(vdev->pdev); in vfio_pci_ioctl_reset()
1244 up_write(&vdev->memory_lock); in vfio_pci_ioctl_reset()
1250 struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_get_pci_hot_reset_info() argument
1270 if (!pci_probe_reset_slot(vdev->pdev->slot)) in vfio_pci_ioctl_get_pci_hot_reset_info()
1272 else if (pci_probe_reset_bus(vdev->pdev->bus)) in vfio_pci_ioctl_get_pci_hot_reset_info()
1275 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, in vfio_pci_ioctl_get_pci_hot_reset_info()
1295 fill.vdev = &vdev->vdev; in vfio_pci_ioctl_get_pci_hot_reset_info()
1297 if (vfio_device_cdev_opened(&vdev->vdev)) in vfio_pci_ioctl_get_pci_hot_reset_info()
1301 mutex_lock(&vdev->vdev.dev_set->lock); in vfio_pci_ioctl_get_pci_hot_reset_info()
1302 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs, in vfio_pci_ioctl_get_pci_hot_reset_info()
1304 mutex_unlock(&vdev->vdev.dev_set->lock); in vfio_pci_ioctl_get_pci_hot_reset_info()
1326 vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_pci_hot_reset_groups() argument
1340 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, in vfio_pci_ioctl_pci_hot_reset_groups()
1394 ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info, NULL); in vfio_pci_ioctl_pci_hot_reset_groups()
1404 static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_pci_hot_reset() argument
1418 if (!!hdr.count == vfio_device_cdev_opened(&vdev->vdev)) in vfio_pci_ioctl_pci_hot_reset()
1422 if (!pci_probe_reset_slot(vdev->pdev->slot)) in vfio_pci_ioctl_pci_hot_reset()
1424 else if (pci_probe_reset_bus(vdev->pdev->bus)) in vfio_pci_ioctl_pci_hot_reset()
1428 return vfio_pci_ioctl_pci_hot_reset_groups(vdev, hdr.count, slot, arg); in vfio_pci_ioctl_pci_hot_reset()
1430 return vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, NULL, in vfio_pci_ioctl_pci_hot_reset()
1431 vfio_iommufd_device_ictx(&vdev->vdev)); in vfio_pci_ioctl_pci_hot_reset()
1434 static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev, in vfio_pci_ioctl_ioeventfd() argument
1455 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count, in vfio_pci_ioctl_ioeventfd()
1462 struct vfio_pci_core_device *vdev = in vfio_pci_core_ioctl() local
1463 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_ioctl()
1468 return vfio_pci_ioctl_get_info(vdev, uarg); in vfio_pci_core_ioctl()
1470 return vfio_pci_ioctl_get_irq_info(vdev, uarg); in vfio_pci_core_ioctl()
1472 return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg); in vfio_pci_core_ioctl()
1474 return vfio_pci_ioctl_get_region_info(vdev, uarg); in vfio_pci_core_ioctl()
1476 return vfio_pci_ioctl_ioeventfd(vdev, uarg); in vfio_pci_core_ioctl()
1478 return vfio_pci_ioctl_pci_hot_reset(vdev, uarg); in vfio_pci_core_ioctl()
1480 return vfio_pci_ioctl_reset(vdev, uarg); in vfio_pci_core_ioctl()
1482 return vfio_pci_ioctl_set_irqs(vdev, uarg); in vfio_pci_core_ioctl()
1492 struct vfio_pci_core_device *vdev = in vfio_pci_core_feature_token() local
1493 container_of(device, struct vfio_pci_core_device, vdev); in vfio_pci_core_feature_token()
1497 if (!vdev->vf_token) in vfio_pci_core_feature_token()
1511 mutex_lock(&vdev->vf_token->lock); in vfio_pci_core_feature_token()
1512 uuid_copy(&vdev->vf_token->uuid, &uuid); in vfio_pci_core_feature_token()
1513 mutex_unlock(&vdev->vf_token->lock); in vfio_pci_core_feature_token()
1536 static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf, in vfio_pci_rw() argument
1542 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) in vfio_pci_rw()
1545 ret = pm_runtime_resume_and_get(&vdev->pdev->dev); in vfio_pci_rw()
1547 pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n", in vfio_pci_rw()
1554 ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); in vfio_pci_rw()
1561 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false); in vfio_pci_rw()
1565 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); in vfio_pci_rw()
1569 ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); in vfio_pci_rw()
1574 ret = vdev->region[index].ops->rw(vdev, buf, in vfio_pci_rw()
1579 pm_runtime_put(&vdev->pdev->dev); in vfio_pci_rw()
1586 struct vfio_pci_core_device *vdev = in vfio_pci_core_read() local
1587 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_read()
1592 return vfio_pci_rw(vdev, buf, count, ppos, false); in vfio_pci_core_read()
1599 struct vfio_pci_core_device *vdev = in vfio_pci_core_write() local
1600 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_write()
1605 return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true); in vfio_pci_core_write()
1609 static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev) in vfio_pci_zap_bars() argument
1611 struct vfio_device *core_vdev = &vdev->vdev; in vfio_pci_zap_bars()
1619 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev) in vfio_pci_zap_and_down_write_memory_lock() argument
1621 down_write(&vdev->memory_lock); in vfio_pci_zap_and_down_write_memory_lock()
1622 vfio_pci_zap_bars(vdev); in vfio_pci_zap_and_down_write_memory_lock()
1625 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev) in vfio_pci_memory_lock_and_enable() argument
1629 down_write(&vdev->memory_lock); in vfio_pci_memory_lock_and_enable()
1630 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd); in vfio_pci_memory_lock_and_enable()
1632 pci_write_config_word(vdev->pdev, PCI_COMMAND, in vfio_pci_memory_lock_and_enable()
1638 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd) in vfio_pci_memory_unlock_and_restore() argument
1640 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd); in vfio_pci_memory_unlock_and_restore()
1641 up_write(&vdev->memory_lock); in vfio_pci_memory_unlock_and_restore()
1646 struct vfio_pci_core_device *vdev = vma->vm_private_data; in vma_to_pfn() local
1653 return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff; in vma_to_pfn()
1660 struct vfio_pci_core_device *vdev = vma->vm_private_data; in vfio_pci_mmap_huge_fault() local
1672 down_read(&vdev->memory_lock); in vfio_pci_mmap_huge_fault()
1674 if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) in vfio_pci_mmap_huge_fault()
1698 up_read(&vdev->memory_lock); in vfio_pci_mmap_huge_fault()
1700 dev_dbg_ratelimited(&vdev->pdev->dev, in vfio_pci_mmap_huge_fault()
1724 struct vfio_pci_core_device *vdev = in vfio_pci_core_mmap() local
1725 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_mmap()
1726 struct pci_dev *pdev = vdev->pdev; in vfio_pci_core_mmap()
1733 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) in vfio_pci_core_mmap()
1741 struct vfio_pci_region *region = vdev->region + regnum; in vfio_pci_core_mmap()
1745 return region->ops->mmap(vdev, region, vma); in vfio_pci_core_mmap()
1750 if (!vdev->bar_mmap_supported[index]) in vfio_pci_core_mmap()
1766 if (!vdev->barmap[index]) { in vfio_pci_core_mmap()
1772 vdev->barmap[index] = pci_iomap(pdev, index, 0); in vfio_pci_core_mmap()
1773 if (!vdev->barmap[index]) { in vfio_pci_core_mmap()
1779 vma->vm_private_data = vdev; in vfio_pci_core_mmap()
1814 struct vfio_pci_core_device *vdev = in vfio_pci_core_request() local
1815 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_request()
1816 struct pci_dev *pdev = vdev->pdev; in vfio_pci_core_request()
1818 mutex_lock(&vdev->igate); in vfio_pci_core_request()
1820 if (vdev->req_trigger) { in vfio_pci_core_request()
1825 eventfd_signal(vdev->req_trigger); in vfio_pci_core_request()
1831 mutex_unlock(&vdev->igate); in vfio_pci_core_request()
1835 static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev, in vfio_pci_validate_vf_token() argument
1863 if (vdev->pdev->is_virtfn) { in vfio_pci_validate_vf_token()
1864 struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev; in vfio_pci_validate_vf_token()
1871 pci_info_ratelimited(vdev->pdev, in vfio_pci_validate_vf_token()
1877 pci_info_ratelimited(vdev->pdev, in vfio_pci_validate_vf_token()
1887 pci_info_ratelimited(vdev->pdev, in vfio_pci_validate_vf_token()
1891 } else if (vdev->vf_token) { in vfio_pci_validate_vf_token()
1892 mutex_lock(&vdev->vf_token->lock); in vfio_pci_validate_vf_token()
1893 if (vdev->vf_token->users) { in vfio_pci_validate_vf_token()
1895 mutex_unlock(&vdev->vf_token->lock); in vfio_pci_validate_vf_token()
1896 pci_info_ratelimited(vdev->pdev, in vfio_pci_validate_vf_token()
1901 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) { in vfio_pci_validate_vf_token()
1902 mutex_unlock(&vdev->vf_token->lock); in vfio_pci_validate_vf_token()
1903 pci_info_ratelimited(vdev->pdev, in vfio_pci_validate_vf_token()
1908 uuid_copy(&vdev->vf_token->uuid, uuid); in vfio_pci_validate_vf_token()
1911 mutex_unlock(&vdev->vf_token->lock); in vfio_pci_validate_vf_token()
1913 pci_info_ratelimited(vdev->pdev, in vfio_pci_validate_vf_token()
1925 struct vfio_pci_core_device *vdev = in vfio_pci_core_match() local
1926 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_match()
1931 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev)))) in vfio_pci_core_match()
1934 if (strlen(buf) > strlen(pci_name(vdev->pdev))) { in vfio_pci_core_match()
1935 buf += strlen(pci_name(vdev->pdev)); in vfio_pci_core_match()
1966 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid); in vfio_pci_core_match()
1977 struct vfio_pci_core_device *vdev = container_of(nb, in vfio_pci_bus_notifier() local
1984 pdev->is_virtfn && physfn == vdev->pdev) { in vfio_pci_bus_notifier()
1985 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n", in vfio_pci_bus_notifier()
1988 vdev->vdev.ops->name); in vfio_pci_bus_notifier()
1991 pdev->is_virtfn && physfn == vdev->pdev) { in vfio_pci_bus_notifier()
1994 if (drv && drv != pci_dev_driver(vdev->pdev)) in vfio_pci_bus_notifier()
1995 pci_warn(vdev->pdev, in vfio_pci_bus_notifier()
1998 pci_dev_driver(vdev->pdev)->name); in vfio_pci_bus_notifier()
2004 static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev) in vfio_pci_vf_init() argument
2006 struct pci_dev *pdev = vdev->pdev; in vfio_pci_vf_init()
2018 physfn = pci_physfn(vdev->pdev); in vfio_pci_vf_init()
2022 vdev->sriov_pf_core_dev = cur; in vfio_pci_vf_init()
2034 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); in vfio_pci_vf_init()
2035 if (!vdev->vf_token) in vfio_pci_vf_init()
2038 mutex_init(&vdev->vf_token->lock); in vfio_pci_vf_init()
2039 uuid_gen(&vdev->vf_token->uuid); in vfio_pci_vf_init()
2041 vdev->nb.notifier_call = vfio_pci_bus_notifier; in vfio_pci_vf_init()
2042 ret = bus_register_notifier(&pci_bus_type, &vdev->nb); in vfio_pci_vf_init()
2044 kfree(vdev->vf_token); in vfio_pci_vf_init()
2050 static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev) in vfio_pci_vf_uninit() argument
2052 if (!vdev->vf_token) in vfio_pci_vf_uninit()
2055 bus_unregister_notifier(&pci_bus_type, &vdev->nb); in vfio_pci_vf_uninit()
2056 WARN_ON(vdev->vf_token->users); in vfio_pci_vf_uninit()
2057 mutex_destroy(&vdev->vf_token->lock); in vfio_pci_vf_uninit()
2058 kfree(vdev->vf_token); in vfio_pci_vf_uninit()
2061 static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev) in vfio_pci_vga_init() argument
2063 struct pci_dev *pdev = vdev->pdev; in vfio_pci_vga_init()
2069 ret = aperture_remove_conflicting_pci_devices(pdev, vdev->vdev.ops->name); in vfio_pci_vga_init()
2080 static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev) in vfio_pci_vga_uninit() argument
2082 struct pci_dev *pdev = vdev->pdev; in vfio_pci_vga_uninit()
2094 struct vfio_pci_core_device *vdev = in vfio_pci_core_init_dev() local
2095 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_init_dev()
2097 vdev->pdev = to_pci_dev(core_vdev->dev); in vfio_pci_core_init_dev()
2098 vdev->irq_type = VFIO_PCI_NUM_IRQS; in vfio_pci_core_init_dev()
2099 mutex_init(&vdev->igate); in vfio_pci_core_init_dev()
2100 spin_lock_init(&vdev->irqlock); in vfio_pci_core_init_dev()
2101 mutex_init(&vdev->ioeventfds_lock); in vfio_pci_core_init_dev()
2102 INIT_LIST_HEAD(&vdev->dummy_resources_list); in vfio_pci_core_init_dev()
2103 INIT_LIST_HEAD(&vdev->ioeventfds_list); in vfio_pci_core_init_dev()
2104 INIT_LIST_HEAD(&vdev->sriov_pfs_item); in vfio_pci_core_init_dev()
2105 init_rwsem(&vdev->memory_lock); in vfio_pci_core_init_dev()
2106 xa_init(&vdev->ctx); in vfio_pci_core_init_dev()
2114 struct vfio_pci_core_device *vdev = in vfio_pci_core_release_dev() local
2115 container_of(core_vdev, struct vfio_pci_core_device, vdev); in vfio_pci_core_release_dev()
2117 mutex_destroy(&vdev->igate); in vfio_pci_core_release_dev()
2118 mutex_destroy(&vdev->ioeventfds_lock); in vfio_pci_core_release_dev()
2119 kfree(vdev->region); in vfio_pci_core_release_dev()
2120 kfree(vdev->pm_save); in vfio_pci_core_release_dev()
2124 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) in vfio_pci_core_register_device() argument
2126 struct pci_dev *pdev = vdev->pdev; in vfio_pci_core_register_device()
2131 if (WARN_ON(vdev != dev_get_drvdata(dev))) in vfio_pci_core_register_device()
2137 if (vdev->vdev.mig_ops) { in vfio_pci_core_register_device()
2138 if (!(vdev->vdev.mig_ops->migration_get_state && in vfio_pci_core_register_device()
2139 vdev->vdev.mig_ops->migration_set_state && in vfio_pci_core_register_device()
2140 vdev->vdev.mig_ops->migration_get_data_size) || in vfio_pci_core_register_device()
2141 !(vdev->vdev.migration_flags & VFIO_MIGRATION_STOP_COPY)) in vfio_pci_core_register_device()
2145 if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start && in vfio_pci_core_register_device()
2146 vdev->vdev.log_ops->log_stop && in vfio_pci_core_register_device()
2147 vdev->vdev.log_ops->log_read_and_clear)) in vfio_pci_core_register_device()
2164 ret = vfio_assign_device_set(&vdev->vdev, vdev); in vfio_pci_core_register_device()
2166 ret = vfio_assign_device_set(&vdev->vdev, pdev->slot); in vfio_pci_core_register_device()
2172 ret = vfio_assign_device_set(&vdev->vdev, pdev->bus); in vfio_pci_core_register_device()
2177 ret = vfio_pci_vf_init(vdev); in vfio_pci_core_register_device()
2180 ret = vfio_pci_vga_init(vdev); in vfio_pci_core_register_device()
2184 vfio_pci_probe_power_state(vdev); in vfio_pci_core_register_device()
2195 vfio_pci_set_power_state(vdev, PCI_D0); in vfio_pci_core_register_device()
2202 ret = vfio_register_group_dev(&vdev->vdev); in vfio_pci_core_register_device()
2213 vfio_pci_vf_uninit(vdev); in vfio_pci_core_register_device()
2218 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) in vfio_pci_core_unregister_device() argument
2220 vfio_pci_core_sriov_configure(vdev, 0); in vfio_pci_core_unregister_device()
2222 vfio_unregister_group_dev(&vdev->vdev); in vfio_pci_core_unregister_device()
2224 vfio_pci_vf_uninit(vdev); in vfio_pci_core_unregister_device()
2225 vfio_pci_vga_uninit(vdev); in vfio_pci_core_unregister_device()
2228 pm_runtime_get_noresume(&vdev->pdev->dev); in vfio_pci_core_unregister_device()
2230 pm_runtime_forbid(&vdev->pdev->dev); in vfio_pci_core_unregister_device()
2237 struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev); in vfio_pci_core_aer_err_detected() local
2239 mutex_lock(&vdev->igate); in vfio_pci_core_aer_err_detected()
2241 if (vdev->err_trigger) in vfio_pci_core_aer_err_detected()
2242 eventfd_signal(vdev->err_trigger); in vfio_pci_core_aer_err_detected()
2244 mutex_unlock(&vdev->igate); in vfio_pci_core_aer_err_detected()
2250 int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, in vfio_pci_core_sriov_configure() argument
2253 struct pci_dev *pdev = vdev->pdev; in vfio_pci_core_sriov_configure()
2261 * The thread that adds the vdev to the list is the only thread in vfio_pci_core_sriov_configure()
2266 if (!list_empty(&vdev->sriov_pfs_item)) { in vfio_pci_core_sriov_configure()
2270 list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs); in vfio_pci_core_sriov_configure()
2287 down_write(&vdev->memory_lock); in vfio_pci_core_sriov_configure()
2288 vfio_pci_set_power_state(vdev, PCI_D0); in vfio_pci_core_sriov_configure()
2290 up_write(&vdev->memory_lock); in vfio_pci_core_sriov_configure()
2305 list_del_init(&vdev->sriov_pfs_item); in vfio_pci_core_sriov_configure()
2317 static bool vfio_dev_in_groups(struct vfio_device *vdev, in vfio_dev_in_groups() argument
2326 if (vfio_file_has_dev(groups->files[i], vdev)) in vfio_dev_in_groups()
2362 vdev.dev_set_list)->pdev; in vfio_pci_dev_set_resettable()
2380 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { in vfio_pci_dev_set_pm_runtime_get()
2390 vdev.dev_set_list) in vfio_pci_dev_set_pm_runtime_get()
2400 struct vfio_pci_core_device *vdev; in vfio_pci_dev_set_hot_reset() local
2421 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) { in vfio_pci_dev_set_hot_reset()
2445 int devid = vfio_iommufd_get_dev_id(&vdev->vdev, in vfio_pci_dev_set_hot_reset()
2450 owned = vfio_dev_in_groups(&vdev->vdev, groups); in vfio_pci_dev_set_hot_reset()
2464 if (!down_write_trylock(&vdev->memory_lock)) { in vfio_pci_dev_set_hot_reset()
2469 vfio_pci_zap_bars(vdev); in vfio_pci_dev_set_hot_reset()
2472 if (!list_entry_is_head(vdev, in vfio_pci_dev_set_hot_reset()
2473 &dev_set->device_list, vdev.dev_set_list)) { in vfio_pci_dev_set_hot_reset()
2474 vdev = list_prev_entry(vdev, vdev.dev_set_list); in vfio_pci_dev_set_hot_reset()
2485 * state (saved locally in 'vdev->pm_save'). in vfio_pci_dev_set_hot_reset()
2487 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) in vfio_pci_dev_set_hot_reset()
2488 vfio_pci_set_power_state(vdev, PCI_D0); in vfio_pci_dev_set_hot_reset()
2492 vdev = list_last_entry(&dev_set->device_list, in vfio_pci_dev_set_hot_reset()
2493 struct vfio_pci_core_device, vdev.dev_set_list); in vfio_pci_dev_set_hot_reset()
2496 list_for_each_entry_from_reverse(vdev, &dev_set->device_list, in vfio_pci_dev_set_hot_reset()
2497 vdev.dev_set_list) in vfio_pci_dev_set_hot_reset()
2498 up_write(&vdev->memory_lock); in vfio_pci_dev_set_hot_reset()
2500 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) in vfio_pci_dev_set_hot_reset()
2501 pm_runtime_put(&vdev->pdev->dev); in vfio_pci_dev_set_hot_reset()
2517 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) in vfio_pci_dev_set_needs_reset()
2553 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { in vfio_pci_dev_set_try_reset()