1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
3
4 #include <linux/interval_tree.h>
5 #include <linux/vfio.h>
6 #include <linux/vmalloc.h>
7
8 #include <linux/pds/pds_common.h>
9 #include <linux/pds/pds_core_if.h>
10 #include <linux/pds/pds_adminq.h>
11
12 #include "vfio_dev.h"
13 #include "cmds.h"
14 #include "dirty.h"
15
16 #define READ_SEQ true
17 #define WRITE_ACK false
18
pds_vfio_dirty_is_enabled(struct pds_vfio_pci_device * pds_vfio)19 bool pds_vfio_dirty_is_enabled(struct pds_vfio_pci_device *pds_vfio)
20 {
21 return pds_vfio->dirty.is_enabled;
22 }
23
pds_vfio_dirty_set_enabled(struct pds_vfio_pci_device * pds_vfio)24 void pds_vfio_dirty_set_enabled(struct pds_vfio_pci_device *pds_vfio)
25 {
26 pds_vfio->dirty.is_enabled = true;
27 }
28
pds_vfio_dirty_set_disabled(struct pds_vfio_pci_device * pds_vfio)29 void pds_vfio_dirty_set_disabled(struct pds_vfio_pci_device *pds_vfio)
30 {
31 pds_vfio->dirty.is_enabled = false;
32 }
33
34 static void
pds_vfio_print_guest_region_info(struct pds_vfio_pci_device * pds_vfio,u8 max_regions)35 pds_vfio_print_guest_region_info(struct pds_vfio_pci_device *pds_vfio,
36 u8 max_regions)
37 {
38 int len = max_regions * sizeof(struct pds_lm_dirty_region_info);
39 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
40 struct device *pdsc_dev = &pci_physfn(pdev)->dev;
41 struct pds_lm_dirty_region_info *region_info;
42 dma_addr_t regions_dma;
43 u8 num_regions;
44 int err;
45
46 region_info = kcalloc(max_regions,
47 sizeof(struct pds_lm_dirty_region_info),
48 GFP_KERNEL);
49 if (!region_info)
50 return;
51
52 regions_dma =
53 dma_map_single(pdsc_dev, region_info, len, DMA_FROM_DEVICE);
54 if (dma_mapping_error(pdsc_dev, regions_dma))
55 goto out_free_region_info;
56
57 err = pds_vfio_dirty_status_cmd(pds_vfio, regions_dma, &max_regions,
58 &num_regions);
59 dma_unmap_single(pdsc_dev, regions_dma, len, DMA_FROM_DEVICE);
60 if (err)
61 goto out_free_region_info;
62
63 for (unsigned int i = 0; i < num_regions; i++)
64 dev_dbg(&pdev->dev,
65 "region_info[%d]: dma_base 0x%llx page_count %u page_size_log2 %u\n",
66 i, le64_to_cpu(region_info[i].dma_base),
67 le32_to_cpu(region_info[i].page_count),
68 region_info[i].page_size_log2);
69
70 out_free_region_info:
71 kfree(region_info);
72 }
73
pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_region * region,unsigned long bytes)74 static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_region *region,
75 unsigned long bytes)
76 {
77 unsigned long *host_seq_bmp, *host_ack_bmp;
78
79 host_seq_bmp = vzalloc(bytes);
80 if (!host_seq_bmp)
81 return -ENOMEM;
82
83 host_ack_bmp = vzalloc(bytes);
84 if (!host_ack_bmp) {
85 bitmap_free(host_seq_bmp);
86 return -ENOMEM;
87 }
88
89 region->host_seq = host_seq_bmp;
90 region->host_ack = host_ack_bmp;
91 region->bmp_bytes = bytes;
92
93 return 0;
94 }
95
pds_vfio_dirty_free_bitmaps(struct pds_vfio_dirty * dirty)96 static void pds_vfio_dirty_free_bitmaps(struct pds_vfio_dirty *dirty)
97 {
98 if (!dirty->regions)
99 return;
100
101 for (int i = 0; i < dirty->num_regions; i++) {
102 struct pds_vfio_region *region = &dirty->regions[i];
103
104 vfree(region->host_seq);
105 vfree(region->host_ack);
106 region->host_seq = NULL;
107 region->host_ack = NULL;
108 region->bmp_bytes = 0;
109 }
110 }
111
__pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device * pds_vfio,struct pds_vfio_region * region)112 static void __pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio,
113 struct pds_vfio_region *region)
114 {
115 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
116 struct device *pdsc_dev = &pci_physfn(pdev)->dev;
117
118 dma_unmap_single(pdsc_dev, region->sgl_addr,
119 region->num_sge * sizeof(struct pds_lm_sg_elem),
120 DMA_BIDIRECTIONAL);
121 kfree(region->sgl);
122
123 region->num_sge = 0;
124 region->sgl = NULL;
125 region->sgl_addr = 0;
126 }
127
pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device * pds_vfio)128 static void pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio)
129 {
130 struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
131
132 if (!dirty->regions)
133 return;
134
135 for (int i = 0; i < dirty->num_regions; i++) {
136 struct pds_vfio_region *region = &dirty->regions[i];
137
138 if (region->sgl)
139 __pds_vfio_dirty_free_sgl(pds_vfio, region);
140 }
141 }
142
pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device * pds_vfio,struct pds_vfio_region * region,u32 page_count)143 static int pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio,
144 struct pds_vfio_region *region,
145 u32 page_count)
146 {
147 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
148 struct device *pdsc_dev = &pci_physfn(pdev)->dev;
149 struct pds_lm_sg_elem *sgl;
150 dma_addr_t sgl_addr;
151 size_t sgl_size;
152 u32 max_sge;
153
154 max_sge = DIV_ROUND_UP(page_count, PAGE_SIZE * 8);
155 sgl_size = max_sge * sizeof(struct pds_lm_sg_elem);
156
157 sgl = kzalloc(sgl_size, GFP_KERNEL);
158 if (!sgl)
159 return -ENOMEM;
160
161 sgl_addr = dma_map_single(pdsc_dev, sgl, sgl_size, DMA_BIDIRECTIONAL);
162 if (dma_mapping_error(pdsc_dev, sgl_addr)) {
163 kfree(sgl);
164 return -EIO;
165 }
166
167 region->sgl = sgl;
168 region->num_sge = max_sge;
169 region->sgl_addr = sgl_addr;
170
171 return 0;
172 }
173
pds_vfio_dirty_free_regions(struct pds_vfio_dirty * dirty)174 static void pds_vfio_dirty_free_regions(struct pds_vfio_dirty *dirty)
175 {
176 vfree(dirty->regions);
177 dirty->regions = NULL;
178 dirty->num_regions = 0;
179 }
180
pds_vfio_dirty_alloc_regions(struct pds_vfio_pci_device * pds_vfio,struct pds_lm_dirty_region_info * region_info,u64 region_page_size,u8 num_regions)181 static int pds_vfio_dirty_alloc_regions(struct pds_vfio_pci_device *pds_vfio,
182 struct pds_lm_dirty_region_info *region_info,
183 u64 region_page_size, u8 num_regions)
184 {
185 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
186 struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
187 u32 dev_bmp_offset_byte = 0;
188 int err;
189
190 dirty->regions = vcalloc(num_regions, sizeof(struct pds_vfio_region));
191 if (!dirty->regions)
192 return -ENOMEM;
193 dirty->num_regions = num_regions;
194
195 for (int i = 0; i < num_regions; i++) {
196 struct pds_lm_dirty_region_info *ri = ®ion_info[i];
197 struct pds_vfio_region *region = &dirty->regions[i];
198 u64 region_size, region_start;
199 u32 page_count;
200
201 /* page_count might be adjusted by the device */
202 page_count = le32_to_cpu(ri->page_count);
203 region_start = le64_to_cpu(ri->dma_base);
204 region_size = page_count * region_page_size;
205
206 err = pds_vfio_dirty_alloc_bitmaps(region,
207 page_count / BITS_PER_BYTE);
208 if (err) {
209 dev_err(&pdev->dev, "Failed to alloc dirty bitmaps: %pe\n",
210 ERR_PTR(err));
211 goto out_free_regions;
212 }
213
214 err = pds_vfio_dirty_alloc_sgl(pds_vfio, region, page_count);
215 if (err) {
216 dev_err(&pdev->dev, "Failed to alloc dirty sg lists: %pe\n",
217 ERR_PTR(err));
218 goto out_free_regions;
219 }
220
221 region->size = region_size;
222 region->start = region_start;
223 region->page_size = region_page_size;
224 region->dev_bmp_offset_start_byte = dev_bmp_offset_byte;
225
226 dev_bmp_offset_byte += page_count / BITS_PER_BYTE;
227 if (dev_bmp_offset_byte % BITS_PER_BYTE) {
228 dev_err(&pdev->dev, "Device bitmap offset is mis-aligned\n");
229 err = -EINVAL;
230 goto out_free_regions;
231 }
232 }
233
234 return 0;
235
236 out_free_regions:
237 pds_vfio_dirty_free_bitmaps(dirty);
238 pds_vfio_dirty_free_sgl(pds_vfio);
239 pds_vfio_dirty_free_regions(dirty);
240
241 return err;
242 }
243
pds_vfio_dirty_enable(struct pds_vfio_pci_device * pds_vfio,struct rb_root_cached * ranges,u32 nnodes,u64 * page_size)244 static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio,
245 struct rb_root_cached *ranges, u32 nnodes,
246 u64 *page_size)
247 {
248 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
249 struct device *pdsc_dev = &pci_physfn(pdev)->dev;
250 struct pds_lm_dirty_region_info *region_info;
251 struct interval_tree_node *node = NULL;
252 u64 region_page_size = *page_size;
253 u8 max_regions = 0, num_regions;
254 dma_addr_t regions_dma = 0;
255 u32 num_ranges = nnodes;
256 int err;
257 u16 len;
258
259 dev_dbg(&pdev->dev, "vf%u: Start dirty page tracking\n",
260 pds_vfio->vf_id);
261
262 if (pds_vfio_dirty_is_enabled(pds_vfio))
263 return -EINVAL;
264
265 /* find if dirty tracking is disabled, i.e. num_regions == 0 */
266 err = pds_vfio_dirty_status_cmd(pds_vfio, 0, &max_regions,
267 &num_regions);
268 if (err < 0) {
269 dev_err(&pdev->dev, "Failed to get dirty status, err %pe\n",
270 ERR_PTR(err));
271 return err;
272 } else if (num_regions) {
273 dev_err(&pdev->dev,
274 "Dirty tracking already enabled for %d regions\n",
275 num_regions);
276 return -EEXIST;
277 } else if (!max_regions) {
278 dev_err(&pdev->dev,
279 "Device doesn't support dirty tracking, max_regions %d\n",
280 max_regions);
281 return -EOPNOTSUPP;
282 }
283
284 if (num_ranges > max_regions) {
285 vfio_combine_iova_ranges(ranges, nnodes, max_regions);
286 num_ranges = max_regions;
287 }
288
289 region_info = kcalloc(num_ranges, sizeof(*region_info), GFP_KERNEL);
290 if (!region_info)
291 return -ENOMEM;
292 len = num_ranges * sizeof(*region_info);
293
294 node = interval_tree_iter_first(ranges, 0, ULONG_MAX);
295 if (!node)
296 return -EINVAL;
297 for (int i = 0; i < num_ranges; i++) {
298 struct pds_lm_dirty_region_info *ri = ®ion_info[i];
299 u64 region_size = node->last - node->start + 1;
300 u64 region_start = node->start;
301 u32 page_count;
302
303 page_count = DIV_ROUND_UP(region_size, region_page_size);
304
305 ri->dma_base = cpu_to_le64(region_start);
306 ri->page_count = cpu_to_le32(page_count);
307 ri->page_size_log2 = ilog2(region_page_size);
308
309 dev_dbg(&pdev->dev,
310 "region_info[%d]: region_start 0x%llx region_end 0x%lx region_size 0x%llx page_count %u page_size %llu\n",
311 i, region_start, node->last, region_size, page_count,
312 region_page_size);
313
314 node = interval_tree_iter_next(node, 0, ULONG_MAX);
315 }
316
317 regions_dma = dma_map_single(pdsc_dev, (void *)region_info, len,
318 DMA_BIDIRECTIONAL);
319 if (dma_mapping_error(pdsc_dev, regions_dma)) {
320 err = -ENOMEM;
321 goto out_free_region_info;
322 }
323
324 err = pds_vfio_dirty_enable_cmd(pds_vfio, regions_dma, num_ranges);
325 dma_unmap_single(pdsc_dev, regions_dma, len, DMA_BIDIRECTIONAL);
326 if (err)
327 goto out_free_region_info;
328
329 err = pds_vfio_dirty_alloc_regions(pds_vfio, region_info,
330 region_page_size, num_ranges);
331 if (err) {
332 dev_err(&pdev->dev,
333 "Failed to allocate %d regions for tracking dirty regions: %pe\n",
334 num_regions, ERR_PTR(err));
335 goto out_dirty_disable;
336 }
337
338 pds_vfio_dirty_set_enabled(pds_vfio);
339
340 pds_vfio_print_guest_region_info(pds_vfio, max_regions);
341
342 kfree(region_info);
343
344 return 0;
345
346 out_dirty_disable:
347 pds_vfio_dirty_disable_cmd(pds_vfio);
348 out_free_region_info:
349 kfree(region_info);
350 return err;
351 }
352
pds_vfio_dirty_disable(struct pds_vfio_pci_device * pds_vfio,bool send_cmd)353 void pds_vfio_dirty_disable(struct pds_vfio_pci_device *pds_vfio, bool send_cmd)
354 {
355 if (pds_vfio_dirty_is_enabled(pds_vfio)) {
356 pds_vfio_dirty_set_disabled(pds_vfio);
357 if (send_cmd)
358 pds_vfio_dirty_disable_cmd(pds_vfio);
359 pds_vfio_dirty_free_sgl(pds_vfio);
360 pds_vfio_dirty_free_bitmaps(&pds_vfio->dirty);
361 pds_vfio_dirty_free_regions(&pds_vfio->dirty);
362 }
363
364 if (send_cmd)
365 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE);
366 }
367
pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device * pds_vfio,struct pds_vfio_region * region,unsigned long * seq_ack_bmp,u32 offset,u32 bmp_bytes,bool read_seq)368 static int pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device *pds_vfio,
369 struct pds_vfio_region *region,
370 unsigned long *seq_ack_bmp, u32 offset,
371 u32 bmp_bytes, bool read_seq)
372 {
373 const char *bmp_type_str = read_seq ? "read_seq" : "write_ack";
374 u8 dma_dir = read_seq ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
375 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
376 struct device *pdsc_dev = &pci_physfn(pdev)->dev;
377 unsigned long long npages;
378 struct sg_table sg_table;
379 struct scatterlist *sg;
380 struct page **pages;
381 u32 page_offset;
382 const void *bmp;
383 size_t size;
384 u16 num_sge;
385 int err;
386 int i;
387
388 bmp = (void *)((u64)seq_ack_bmp + offset);
389 page_offset = offset_in_page(bmp);
390 bmp -= page_offset;
391
392 /*
393 * Start and end of bitmap section to seq/ack might not be page
394 * aligned, so use the page_offset to account for that so there
395 * will be enough pages to represent the bmp_bytes
396 */
397 npages = DIV_ROUND_UP_ULL(bmp_bytes + page_offset, PAGE_SIZE);
398 pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
399 if (!pages)
400 return -ENOMEM;
401
402 for (unsigned long long i = 0; i < npages; i++) {
403 struct page *page = vmalloc_to_page(bmp);
404
405 if (!page) {
406 err = -EFAULT;
407 goto out_free_pages;
408 }
409
410 pages[i] = page;
411 bmp += PAGE_SIZE;
412 }
413
414 err = sg_alloc_table_from_pages(&sg_table, pages, npages, page_offset,
415 bmp_bytes, GFP_KERNEL);
416 if (err)
417 goto out_free_pages;
418
419 err = dma_map_sgtable(pdsc_dev, &sg_table, dma_dir, 0);
420 if (err)
421 goto out_free_sg_table;
422
423 for_each_sgtable_dma_sg(&sg_table, sg, i) {
424 struct pds_lm_sg_elem *sg_elem = ®ion->sgl[i];
425
426 sg_elem->addr = cpu_to_le64(sg_dma_address(sg));
427 sg_elem->len = cpu_to_le32(sg_dma_len(sg));
428 }
429
430 num_sge = sg_table.nents;
431 size = num_sge * sizeof(struct pds_lm_sg_elem);
432 offset += region->dev_bmp_offset_start_byte;
433 dma_sync_single_for_device(pdsc_dev, region->sgl_addr, size, dma_dir);
434 err = pds_vfio_dirty_seq_ack_cmd(pds_vfio, region->sgl_addr, num_sge,
435 offset, bmp_bytes, read_seq);
436 if (err)
437 dev_err(&pdev->dev,
438 "Dirty bitmap %s failed offset %u bmp_bytes %u num_sge %u DMA 0x%llx: %pe\n",
439 bmp_type_str, offset, bmp_bytes,
440 num_sge, region->sgl_addr, ERR_PTR(err));
441 dma_sync_single_for_cpu(pdsc_dev, region->sgl_addr, size, dma_dir);
442
443 dma_unmap_sgtable(pdsc_dev, &sg_table, dma_dir, 0);
444 out_free_sg_table:
445 sg_free_table(&sg_table);
446 out_free_pages:
447 kfree(pages);
448
449 return err;
450 }
451
pds_vfio_dirty_write_ack(struct pds_vfio_pci_device * pds_vfio,struct pds_vfio_region * region,u32 offset,u32 len)452 static int pds_vfio_dirty_write_ack(struct pds_vfio_pci_device *pds_vfio,
453 struct pds_vfio_region *region,
454 u32 offset, u32 len)
455 {
456
457 return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_ack,
458 offset, len, WRITE_ACK);
459 }
460
pds_vfio_dirty_read_seq(struct pds_vfio_pci_device * pds_vfio,struct pds_vfio_region * region,u32 offset,u32 len)461 static int pds_vfio_dirty_read_seq(struct pds_vfio_pci_device *pds_vfio,
462 struct pds_vfio_region *region,
463 u32 offset, u32 len)
464 {
465 return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_seq,
466 offset, len, READ_SEQ);
467 }
468
pds_vfio_dirty_process_bitmaps(struct pds_vfio_pci_device * pds_vfio,struct pds_vfio_region * region,struct iova_bitmap * dirty_bitmap,u32 bmp_offset,u32 len_bytes)469 static int pds_vfio_dirty_process_bitmaps(struct pds_vfio_pci_device *pds_vfio,
470 struct pds_vfio_region *region,
471 struct iova_bitmap *dirty_bitmap,
472 u32 bmp_offset, u32 len_bytes)
473 {
474 u64 page_size = region->page_size;
475 u64 region_start = region->start;
476 u32 bmp_offset_bit;
477 __le64 *seq, *ack;
478 int dword_count;
479
480 dword_count = len_bytes / sizeof(u64);
481 seq = (__le64 *)((u64)region->host_seq + bmp_offset);
482 ack = (__le64 *)((u64)region->host_ack + bmp_offset);
483 bmp_offset_bit = bmp_offset * 8;
484
485 for (int i = 0; i < dword_count; i++) {
486 u64 xor = le64_to_cpu(seq[i]) ^ le64_to_cpu(ack[i]);
487
488 /* prepare for next write_ack call */
489 ack[i] = seq[i];
490
491 for (u8 bit_i = 0; bit_i < BITS_PER_TYPE(u64); ++bit_i) {
492 if (xor & BIT(bit_i)) {
493 u64 abs_bit_i = bmp_offset_bit +
494 i * BITS_PER_TYPE(u64) + bit_i;
495 u64 addr = abs_bit_i * page_size + region_start;
496
497 iova_bitmap_set(dirty_bitmap, addr, page_size);
498 }
499 }
500 }
501
502 return 0;
503 }
504
505 static struct pds_vfio_region *
pds_vfio_get_region(struct pds_vfio_pci_device * pds_vfio,unsigned long iova)506 pds_vfio_get_region(struct pds_vfio_pci_device *pds_vfio, unsigned long iova)
507 {
508 struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
509
510 for (int i = 0; i < dirty->num_regions; i++) {
511 struct pds_vfio_region *region = &dirty->regions[i];
512
513 if (iova >= region->start &&
514 iova < (region->start + region->size))
515 return region;
516 }
517
518 return NULL;
519 }
520
pds_vfio_dirty_sync(struct pds_vfio_pci_device * pds_vfio,struct iova_bitmap * dirty_bitmap,unsigned long iova,unsigned long length)521 static int pds_vfio_dirty_sync(struct pds_vfio_pci_device *pds_vfio,
522 struct iova_bitmap *dirty_bitmap,
523 unsigned long iova, unsigned long length)
524 {
525 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
526 struct pds_vfio_region *region;
527 u64 bmp_offset, bmp_bytes;
528 u64 bitmap_size, pages;
529 int err;
530
531 dev_dbg(dev, "vf%u: Get dirty page bitmap\n", pds_vfio->vf_id);
532
533 if (!pds_vfio_dirty_is_enabled(pds_vfio)) {
534 dev_err(dev, "vf%u: Sync failed, dirty tracking is disabled\n",
535 pds_vfio->vf_id);
536 return -EINVAL;
537 }
538
539 region = pds_vfio_get_region(pds_vfio, iova);
540 if (!region) {
541 dev_err(dev, "vf%u: Failed to find region that contains iova 0x%lx length 0x%lx\n",
542 pds_vfio->vf_id, iova, length);
543 return -EINVAL;
544 }
545
546 pages = DIV_ROUND_UP(length, region->page_size);
547 bitmap_size =
548 round_up(pages, sizeof(u64) * BITS_PER_BYTE) / BITS_PER_BYTE;
549
550 dev_dbg(dev,
551 "vf%u: iova 0x%lx length %lu page_size %llu pages %llu bitmap_size %llu\n",
552 pds_vfio->vf_id, iova, length, region->page_size,
553 pages, bitmap_size);
554
555 if (!length || ((iova - region->start + length) > region->size)) {
556 dev_err(dev, "Invalid iova 0x%lx and/or length 0x%lx to sync\n",
557 iova, length);
558 return -EINVAL;
559 }
560
561 /* bitmap is modified in 64 bit chunks */
562 bmp_bytes = ALIGN(DIV_ROUND_UP(length / region->page_size,
563 sizeof(u64)), sizeof(u64));
564 if (bmp_bytes != bitmap_size) {
565 dev_err(dev,
566 "Calculated bitmap bytes %llu not equal to bitmap size %llu\n",
567 bmp_bytes, bitmap_size);
568 return -EINVAL;
569 }
570
571 if (bmp_bytes > region->bmp_bytes) {
572 dev_err(dev,
573 "Calculated bitmap bytes %llu larger than region's cached bmp_bytes %llu\n",
574 bmp_bytes, region->bmp_bytes);
575 return -EINVAL;
576 }
577
578 bmp_offset = DIV_ROUND_UP((iova - region->start) /
579 region->page_size, sizeof(u64));
580
581 dev_dbg(dev,
582 "Syncing dirty bitmap, iova 0x%lx length 0x%lx, bmp_offset %llu bmp_bytes %llu\n",
583 iova, length, bmp_offset, bmp_bytes);
584
585 err = pds_vfio_dirty_read_seq(pds_vfio, region, bmp_offset, bmp_bytes);
586 if (err)
587 return err;
588
589 err = pds_vfio_dirty_process_bitmaps(pds_vfio, region, dirty_bitmap,
590 bmp_offset, bmp_bytes);
591 if (err)
592 return err;
593
594 err = pds_vfio_dirty_write_ack(pds_vfio, region, bmp_offset, bmp_bytes);
595 if (err)
596 return err;
597
598 return 0;
599 }
600
pds_vfio_dma_logging_report(struct vfio_device * vdev,unsigned long iova,unsigned long length,struct iova_bitmap * dirty)601 int pds_vfio_dma_logging_report(struct vfio_device *vdev, unsigned long iova,
602 unsigned long length, struct iova_bitmap *dirty)
603 {
604 struct pds_vfio_pci_device *pds_vfio =
605 container_of(vdev, struct pds_vfio_pci_device,
606 vfio_coredev.vdev);
607 int err;
608
609 mutex_lock(&pds_vfio->state_mutex);
610 err = pds_vfio_dirty_sync(pds_vfio, dirty, iova, length);
611 mutex_unlock(&pds_vfio->state_mutex);
612
613 return err;
614 }
615
pds_vfio_dma_logging_start(struct vfio_device * vdev,struct rb_root_cached * ranges,u32 nnodes,u64 * page_size)616 int pds_vfio_dma_logging_start(struct vfio_device *vdev,
617 struct rb_root_cached *ranges, u32 nnodes,
618 u64 *page_size)
619 {
620 struct pds_vfio_pci_device *pds_vfio =
621 container_of(vdev, struct pds_vfio_pci_device,
622 vfio_coredev.vdev);
623 int err;
624
625 mutex_lock(&pds_vfio->state_mutex);
626 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_IN_PROGRESS);
627 err = pds_vfio_dirty_enable(pds_vfio, ranges, nnodes, page_size);
628 mutex_unlock(&pds_vfio->state_mutex);
629
630 return err;
631 }
632
pds_vfio_dma_logging_stop(struct vfio_device * vdev)633 int pds_vfio_dma_logging_stop(struct vfio_device *vdev)
634 {
635 struct pds_vfio_pci_device *pds_vfio =
636 container_of(vdev, struct pds_vfio_pci_device,
637 vfio_coredev.vdev);
638
639 mutex_lock(&pds_vfio->state_mutex);
640 pds_vfio_dirty_disable(pds_vfio, true);
641 mutex_unlock(&pds_vfio->state_mutex);
642
643 return 0;
644 }
645