1 // SPDX-License-Identifier: GPL-2.0-or-later
9  * vp_modern_map_capability - map a part of virtio pci capability
10  * @mdev: the modern virtio-pci device
26 	struct pci_dev *dev = mdev->pci_dev;  in vp_modern_map_capability()
40 	if (bar >= PCI_STD_NUM_BARS || !(mdev->modern_bars & (1 << bar))) {  in vp_modern_map_capability()
41 		dev_err(&dev->dev,  in vp_modern_map_capability()
47 		dev_err(&dev->dev,  in vp_modern_map_capability()
53 	if (length - start < minlen) {  in vp_modern_map_capability()
54 		dev_err(&dev->dev,  in vp_modern_map_capability()
60 	length -= start;  in vp_modern_map_capability()
63 		dev_err(&dev->dev,  in vp_modern_map_capability()
64 			"virtio_pci: map wrap-around %u+%u\n",  in vp_modern_map_capability()
71 	if (offset & (align - 1)) {  in vp_modern_map_capability()
72 		dev_err(&dev->dev,  in vp_modern_map_capability()
86 		dev_err(&dev->dev,  in vp_modern_map_capability()
87 			"virtio_pci: map virtio %zu@%u "  in vp_modern_map_capability()
96 		dev_err(&dev->dev,  in vp_modern_map_capability()
97 			"virtio_pci: unable to map virtio %u@%u on bar %i\n",  in vp_modern_map_capability()
106  * virtio_pci_find_capability - walk capabilities to find device info.
107  * @dev: the pci device
217  * vp_modern_probe: probe the modern virtio pci device, note that the
218  * caller is required to enable PCI device before calling this function.
219  * @mdev: the modern virtio-pci device
225 	struct pci_dev *pci_dev = mdev->pci_dev;  in vp_modern_probe()
226 	int err, common, isr, notify, device;  in vp_modern_probe()  local
233 	if (mdev->device_id_check) {  in vp_modern_probe()
234 		devid = mdev->device_id_check(pci_dev);  in vp_modern_probe()
237 		mdev->id.device = devid;  in vp_modern_probe()
240 		if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)  in vp_modern_probe()
241 			return -ENODEV;  in vp_modern_probe()
243 		if (pci_dev->device < 0x1040) {  in vp_modern_probe()
244 			/* Transitional devices: use the PCI subsystem device id as  in vp_modern_probe()
245 			 * virtio device id, same as legacy driver always did.  in vp_modern_probe()
247 			mdev->id.device = pci_dev->subsystem_device;  in vp_modern_probe()
249 			/* Modern devices: simply use PCI device id, but start from 0x1040. */  in vp_modern_probe()
250 			mdev->id.device = pci_dev->device - 0x1040;  in vp_modern_probe()
253 	mdev->id.vendor = pci_dev->subsystem_vendor;  in vp_modern_probe()
258 					    &mdev->modern_bars);  in vp_modern_probe()
260 		dev_info(&pci_dev->dev,  in vp_modern_probe()
262 		return -ENODEV;  in vp_modern_probe()
268 					 &mdev->modern_bars);  in vp_modern_probe()
271 					    &mdev->modern_bars);  in vp_modern_probe()
273 		dev_err(&pci_dev->dev,  in vp_modern_probe()
276 		return -EINVAL;  in vp_modern_probe()
279 	err = dma_set_mask_and_coherent(&pci_dev->dev,  in vp_modern_probe()
280 					mdev->dma_mask ? : DMA_BIT_MASK(64));  in vp_modern_probe()
282 		err = dma_set_mask_and_coherent(&pci_dev->dev,  in vp_modern_probe()
285 …dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA.  Trying to continue, but this migh…  in vp_modern_probe()
287 	/* Device capability is only mandatory for devices that have  in vp_modern_probe()
288 	 * device-specific configuration.  in vp_modern_probe()
290 	device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG,  in vp_modern_probe()
292 					    &mdev->modern_bars);  in vp_modern_probe()
294 	err = pci_request_selected_regions(pci_dev, mdev->modern_bars,  in vp_modern_probe()
295 					   "virtio-pci-modern");  in vp_modern_probe()
299 	err = -EINVAL;  in vp_modern_probe()
300 	mdev->common = vp_modern_map_capability(mdev, common,  in vp_modern_probe()
304 			      &mdev->common_len, NULL);  in vp_modern_probe()
305 	if (!mdev->common)  in vp_modern_probe()
307 	mdev->isr = vp_modern_map_capability(mdev, isr, sizeof(u8), 1,  in vp_modern_probe()
310 	if (!mdev->isr)  in vp_modern_probe()
317 			      &mdev->notify_offset_multiplier);  in vp_modern_probe()
334 		mdev->notify_base = vp_modern_map_capability(mdev, notify,  in vp_modern_probe()
337 							     &mdev->notify_len,  in vp_modern_probe()
338 							     &mdev->notify_pa);  in vp_modern_probe()
339 		if (!mdev->notify_base)  in vp_modern_probe()
342 		mdev->notify_map_cap = notify;  in vp_modern_probe()
348 	if (device) {  in vp_modern_probe()
349 		mdev->device = vp_modern_map_capability(mdev, device, 0, 4,  in vp_modern_probe()
351 							&mdev->device_len,  in vp_modern_probe()
353 		if (!mdev->device)  in vp_modern_probe()
360 	if (mdev->notify_base)  in vp_modern_probe()
361 		pci_iounmap(pci_dev, mdev->notify_base);  in vp_modern_probe()
363 	pci_iounmap(pci_dev, mdev->isr);  in vp_modern_probe()
365 	pci_iounmap(pci_dev, mdev->common);  in vp_modern_probe()
367 	pci_release_selected_regions(pci_dev, mdev->modern_bars);  in vp_modern_probe()
373  * vp_modern_remove: remove and cleanup the modern virtio pci device
374  * @mdev: the modern virtio-pci device
378 	struct pci_dev *pci_dev = mdev->pci_dev;  in vp_modern_remove()
380 	if (mdev->device)  in vp_modern_remove()
381 		pci_iounmap(pci_dev, mdev->device);  in vp_modern_remove()
382 	if (mdev->notify_base)  in vp_modern_remove()
383 		pci_iounmap(pci_dev, mdev->notify_base);  in vp_modern_remove()
384 	pci_iounmap(pci_dev, mdev->isr);  in vp_modern_remove()
385 	pci_iounmap(pci_dev, mdev->common);  in vp_modern_remove()
386 	pci_release_selected_regions(pci_dev, mdev->modern_bars);  in vp_modern_remove()
391  * vp_modern_get_features - get features from device
392  * @mdev: the modern virtio-pci device
394  * Returns the features read from the device
398 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_get_features()
402 	vp_iowrite32(0, &cfg->device_feature_select);  in vp_modern_get_features()
403 	features = vp_ioread32(&cfg->device_feature);  in vp_modern_get_features()
404 	vp_iowrite32(1, &cfg->device_feature_select);  in vp_modern_get_features()
405 	features |= ((u64)vp_ioread32(&cfg->device_feature) << 32);  in vp_modern_get_features()
412  * vp_modern_get_driver_features - get driver features from device
413  * @mdev: the modern virtio-pci device
415  * Returns the driver features read from the device
419 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_get_driver_features()
423 	vp_iowrite32(0, &cfg->guest_feature_select);  in vp_modern_get_driver_features()
424 	features = vp_ioread32(&cfg->guest_feature);  in vp_modern_get_driver_features()
425 	vp_iowrite32(1, &cfg->guest_feature_select);  in vp_modern_get_driver_features()
426 	features |= ((u64)vp_ioread32(&cfg->guest_feature) << 32);  in vp_modern_get_driver_features()
433  * vp_modern_set_features - set features to device
434  * @mdev: the modern virtio-pci device
435  * @features: the features set to device
440 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_set_features()
442 	vp_iowrite32(0, &cfg->guest_feature_select);  in vp_modern_set_features()
443 	vp_iowrite32((u32)features, &cfg->guest_feature);  in vp_modern_set_features()
444 	vp_iowrite32(1, &cfg->guest_feature_select);  in vp_modern_set_features()
445 	vp_iowrite32(features >> 32, &cfg->guest_feature);  in vp_modern_set_features()
450  * vp_modern_generation - get the device genreation
451  * @mdev: the modern virtio-pci device
453  * Returns the genreation read from device
457 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_generation()
459 	return vp_ioread8(&cfg->config_generation);  in vp_modern_generation()
464  * vp_modern_get_status - get the device status
465  * @mdev: the modern virtio-pci device
467  * Returns the status read from device
471 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_get_status()
473 	return vp_ioread8(&cfg->device_status);  in vp_modern_get_status()
478  * vp_modern_set_status - set status to device
479  * @mdev: the modern virtio-pci device
480  * @status: the status set to device
485 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_set_status()
488 	 * Per memory-barriers.txt, wmb() is not needed to guarantee  in vp_modern_set_status()
492 	vp_iowrite8(status, &cfg->device_status);  in vp_modern_set_status()
497  * vp_modern_get_queue_reset - get the queue reset status
498  * @mdev: the modern virtio-pci device
505 	cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;  in vp_modern_get_queue_reset()
507 	vp_iowrite16(index, &cfg->cfg.queue_select);  in vp_modern_get_queue_reset()
508 	return vp_ioread16(&cfg->queue_reset);  in vp_modern_get_queue_reset()
513  * vp_modern_set_queue_reset - reset the queue
514  * @mdev: the modern virtio-pci device
521 	cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;  in vp_modern_set_queue_reset()
523 	vp_iowrite16(index, &cfg->cfg.queue_select);  in vp_modern_set_queue_reset()
524 	vp_iowrite16(1, &cfg->queue_reset);  in vp_modern_set_queue_reset()
526 	while (vp_ioread16(&cfg->queue_reset))  in vp_modern_set_queue_reset()
529 	while (vp_ioread16(&cfg->cfg.queue_enable))  in vp_modern_set_queue_reset()
535  * vp_modern_queue_vector - set the MSIX vector for a specific virtqueue
536  * @mdev: the modern virtio-pci device
540  * Returns the config vector read from the device
545 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_queue_vector()
547 	vp_iowrite16(index, &cfg->queue_select);  in vp_modern_queue_vector()
548 	vp_iowrite16(vector, &cfg->queue_msix_vector);  in vp_modern_queue_vector()
549 	/* Flush the write out to device */  in vp_modern_queue_vector()
550 	return vp_ioread16(&cfg->queue_msix_vector);  in vp_modern_queue_vector()
555  * vp_modern_config_vector - set the vector for config interrupt
556  * @mdev: the modern virtio-pci device
559  * Returns the config vector read from the device
564 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_config_vector()
567 	vp_iowrite16(vector, &cfg->msix_config);  in vp_modern_config_vector()
569 	/* Will also flush the write out to device */  in vp_modern_config_vector()
570 	return vp_ioread16(&cfg->msix_config);  in vp_modern_config_vector()
575  * vp_modern_queue_address - set the virtqueue address
576  * @mdev: the modern virtio-pci device
580  * @device_addr: address of the device area
586 	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;  in vp_modern_queue_address()
588 	vp_iowrite16(index, &cfg->queue_select);  in vp_modern_queue_address()
590 	vp_iowrite64_twopart(desc_addr, &cfg->queue_desc_lo,  in vp_modern_queue_address()
591 			     &cfg->queue_desc_hi);  in vp_modern_queue_address()
592 	vp_iowrite64_twopart(driver_addr, &cfg->queue_avail_lo,  in vp_modern_queue_address()
593 			     &cfg->queue_avail_hi);  in vp_modern_queue_address()
594 	vp_iowrite64_twopart(device_addr, &cfg->queue_used_lo,  in vp_modern_queue_address()
595 			     &cfg->queue_used_hi);  in vp_modern_queue_address()
600  * vp_modern_set_queue_enable - enable a virtqueue
601  * @mdev: the modern virtio-pci device
608 	vp_iowrite16(index, &mdev->common->queue_select);  in vp_modern_set_queue_enable()
609 	vp_iowrite16(enable, &mdev->common->queue_enable);  in vp_modern_set_queue_enable()
614  * vp_modern_get_queue_enable - enable a virtqueue
615  * @mdev: the modern virtio-pci device
623 	vp_iowrite16(index, &mdev->common->queue_select);  in vp_modern_get_queue_enable()
625 	return vp_ioread16(&mdev->common->queue_enable);  in vp_modern_get_queue_enable()
630  * vp_modern_set_queue_size - set size for a virtqueue
631  * @mdev: the modern virtio-pci device
638 	vp_iowrite16(index, &mdev->common->queue_select);  in vp_modern_set_queue_size()
639 	vp_iowrite16(size, &mdev->common->queue_size);  in vp_modern_set_queue_size()
645  * vp_modern_get_queue_size - get size for a virtqueue
646  * @mdev: the modern virtio-pci device
654 	vp_iowrite16(index, &mdev->common->queue_select);  in vp_modern_get_queue_size()
656 	return vp_ioread16(&mdev->common->queue_size);  in vp_modern_get_queue_size()
662  * vp_modern_get_num_queues - get the number of virtqueues
663  * @mdev: the modern virtio-pci device
669 	return vp_ioread16(&mdev->common->num_queues);  in vp_modern_get_num_queues()
674  * vp_modern_get_queue_notify_off - get notification offset for a virtqueue
675  * @mdev: the modern virtio-pci device
683 	vp_iowrite16(index, &mdev->common->queue_select);  in vp_modern_get_queue_notify_off()
685 	return vp_ioread16(&mdev->common->queue_notify_off);  in vp_modern_get_queue_notify_off()
689  * vp_modern_map_vq_notify - map notification area for a
691  * @mdev: the modern virtio-pci device
702 	if (mdev->notify_base) {  in vp_modern_map_vq_notify()
704 		if ((u64)off * mdev->notify_offset_multiplier + 2  in vp_modern_map_vq_notify()
705 			> mdev->notify_len) {  in vp_modern_map_vq_notify()
706 			dev_warn(&mdev->pci_dev->dev,  in vp_modern_map_vq_notify()
709 				 off, mdev->notify_offset_multiplier,  in vp_modern_map_vq_notify()
710 				 index, mdev->notify_len);  in vp_modern_map_vq_notify()
714 			*pa = mdev->notify_pa +  in vp_modern_map_vq_notify()
715 			      off * mdev->notify_offset_multiplier;  in vp_modern_map_vq_notify()
716 		return mdev->notify_base + off * mdev->notify_offset_multiplier;  in vp_modern_map_vq_notify()
719 				       mdev->notify_map_cap, 2, 2,  in vp_modern_map_vq_notify()
720 				       off * mdev->notify_offset_multiplier, 2,  in vp_modern_map_vq_notify()
730 	cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;  in vp_modern_avq_num()
731 	return vp_ioread16(&cfg->admin_queue_num);  in vp_modern_avq_num()
739 	cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;  in vp_modern_avq_index()
740 	return vp_ioread16(&cfg->admin_queue_index);  in vp_modern_avq_index()
745 MODULE_DESCRIPTION("Modern Virtio PCI Device");