74 static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_region *region,  in pds_vfio_dirty_alloc_bitmaps()  argument
89 	region->host_seq = host_seq_bmp;  in pds_vfio_dirty_alloc_bitmaps()
90 	region->host_ack = host_ack_bmp;  in pds_vfio_dirty_alloc_bitmaps()
91 	region->bmp_bytes = bytes;  in pds_vfio_dirty_alloc_bitmaps()
102 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_dirty_free_bitmaps()  local
104 		vfree(region->host_seq);  in pds_vfio_dirty_free_bitmaps()
105 		vfree(region->host_ack);  in pds_vfio_dirty_free_bitmaps()
106 		region->host_seq = NULL;  in pds_vfio_dirty_free_bitmaps()
107 		region->host_ack = NULL;  in pds_vfio_dirty_free_bitmaps()
108 		region->bmp_bytes = 0;  in pds_vfio_dirty_free_bitmaps()
113 				      struct pds_vfio_region *region)  in __pds_vfio_dirty_free_sgl()  argument
118 	dma_unmap_single(pdsc_dev, region->sgl_addr,  in __pds_vfio_dirty_free_sgl()
119 			 region->num_sge * sizeof(struct pds_lm_sg_elem),  in __pds_vfio_dirty_free_sgl()
121 	kfree(region->sgl);  in __pds_vfio_dirty_free_sgl()
123 	region->num_sge = 0;  in __pds_vfio_dirty_free_sgl()
124 	region->sgl = NULL;  in __pds_vfio_dirty_free_sgl()
125 	region->sgl_addr = 0;  in __pds_vfio_dirty_free_sgl()
136 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_dirty_free_sgl()  local
138 		if (region->sgl)  in pds_vfio_dirty_free_sgl()
139 			__pds_vfio_dirty_free_sgl(pds_vfio, region);  in pds_vfio_dirty_free_sgl()
144 				    struct pds_vfio_region *region,  in pds_vfio_dirty_alloc_sgl()  argument
167 	region->sgl = sgl;  in pds_vfio_dirty_alloc_sgl()
168 	region->num_sge = max_sge;  in pds_vfio_dirty_alloc_sgl()
169 	region->sgl_addr = sgl_addr;  in pds_vfio_dirty_alloc_sgl()
197 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_dirty_alloc_regions()  local
206 		err = pds_vfio_dirty_alloc_bitmaps(region,  in pds_vfio_dirty_alloc_regions()
214 		err = pds_vfio_dirty_alloc_sgl(pds_vfio, region, page_count);  in pds_vfio_dirty_alloc_regions()
221 		region->size = region_size;  in pds_vfio_dirty_alloc_regions()
222 		region->start = region_start;  in pds_vfio_dirty_alloc_regions()
223 		region->page_size = region_page_size;  in pds_vfio_dirty_alloc_regions()
224 		region->dev_bmp_offset_start_byte = dev_bmp_offset_byte;  in pds_vfio_dirty_alloc_regions()
369 				  struct pds_vfio_region *region,  in pds_vfio_dirty_seq_ack()  argument
424 		struct pds_lm_sg_elem *sg_elem = &region->sgl[i];  in pds_vfio_dirty_seq_ack()
432 	offset += region->dev_bmp_offset_start_byte;  in pds_vfio_dirty_seq_ack()
433 	dma_sync_single_for_device(pdsc_dev, region->sgl_addr, size, dma_dir);  in pds_vfio_dirty_seq_ack()
434 	err = pds_vfio_dirty_seq_ack_cmd(pds_vfio, region->sgl_addr, num_sge,  in pds_vfio_dirty_seq_ack()
440 			num_sge, region->sgl_addr, ERR_PTR(err));  in pds_vfio_dirty_seq_ack()
441 	dma_sync_single_for_cpu(pdsc_dev, region->sgl_addr, size, dma_dir);  in pds_vfio_dirty_seq_ack()
453 				   struct pds_vfio_region *region,  in pds_vfio_dirty_write_ack()  argument
457 	return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_ack,  in pds_vfio_dirty_write_ack()
462 				   struct pds_vfio_region *region,  in pds_vfio_dirty_read_seq()  argument
465 	return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_seq,  in pds_vfio_dirty_read_seq()
470 					  struct pds_vfio_region *region,  in pds_vfio_dirty_process_bitmaps()  argument
474 	u64 page_size = region->page_size;  in pds_vfio_dirty_process_bitmaps()
475 	u64 region_start = region->start;  in pds_vfio_dirty_process_bitmaps()
481 	seq = (__le64 *)((u64)region->host_seq + bmp_offset);  in pds_vfio_dirty_process_bitmaps()
482 	ack = (__le64 *)((u64)region->host_ack + bmp_offset);  in pds_vfio_dirty_process_bitmaps()
511 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_get_region()  local
513 		if (iova >= region->start &&  in pds_vfio_get_region()
514 		    iova < (region->start + region->size))  in pds_vfio_get_region()
515 			return region;  in pds_vfio_get_region()
526 	struct pds_vfio_region *region;  in pds_vfio_dirty_sync()  local
539 	region = pds_vfio_get_region(pds_vfio, iova);  in pds_vfio_dirty_sync()
540 	if (!region) {  in pds_vfio_dirty_sync()
541 		dev_err(dev, "vf%u: Failed to find region that contains iova 0x%lx length 0x%lx\n",  in pds_vfio_dirty_sync()
546 	pages = DIV_ROUND_UP(length, region->page_size);  in pds_vfio_dirty_sync()
552 		pds_vfio->vf_id, iova, length, region->page_size,  in pds_vfio_dirty_sync()
555 	if (!length || ((iova - region->start + length) > region->size)) {  in pds_vfio_dirty_sync()
562 	bmp_bytes = ALIGN(DIV_ROUND_UP(length / region->page_size,  in pds_vfio_dirty_sync()
571 	if (bmp_bytes > region->bmp_bytes) {  in pds_vfio_dirty_sync()
573 			"Calculated bitmap bytes %llu larger than region's cached bmp_bytes %llu\n",  in pds_vfio_dirty_sync()
574 			bmp_bytes, region->bmp_bytes);  in pds_vfio_dirty_sync()
578 	bmp_offset = DIV_ROUND_UP((iova - region->start) /  in pds_vfio_dirty_sync()
579 				  region->page_size, sizeof(u64));  in pds_vfio_dirty_sync()
585 	err = pds_vfio_dirty_read_seq(pds_vfio, region, bmp_offset, bmp_bytes);  in pds_vfio_dirty_sync()
589 	err = pds_vfio_dirty_process_bitmaps(pds_vfio, region, dirty_bitmap,  in pds_vfio_dirty_sync()
594 	err = pds_vfio_dirty_write_ack(pds_vfio, region, bmp_offset, bmp_bytes);  in pds_vfio_dirty_sync()