Lines Matching +full:de +full:- +full:serialized
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (c) 2014-2015, Intel Corporation.
6 * Copyright (c) 2015, Christoph Hellwig <hch@lst.de>.
22 #include <linux/blk-mq.h>
41 return pmem->bb.dev; in to_dev()
46 return to_nd_region(to_dev(pmem)->parent); in to_region()
51 return pmem->phys_addr + offset; in pmem_to_phys()
56 return (offset - pmem->data_offset) >> SECTOR_SHIFT; in to_sect()
61 return (sector << SECTOR_SHIFT) + pmem->data_offset; in to_offset()
71 if (is_vmalloc_addr(pmem->virt_addr)) in pmem_mkpage_present()
93 badblocks_clear(&pmem->bb, sector, blks); in pmem_clear_bb()
94 if (pmem->bb_state) in pmem_clear_bb()
95 sysfs_notify_dirent(pmem->bb_state); in pmem_clear_bb()
106 arch_invalidate_pmem(pmem->virt_addr + offset, len); in __pmem_clear_poison()
133 chunk = min_t(unsigned int, len, PAGE_SIZE - off); in write_pmem()
136 len -= chunk; in write_pmem()
152 chunk = min_t(unsigned int, len, PAGE_SIZE - off); in read_pmem()
157 len -= chunk; in read_pmem()
171 void *pmem_addr = pmem->virt_addr + pmem_off; in pmem_do_read()
173 if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) in pmem_do_read()
186 void *pmem_addr = pmem->virt_addr + pmem_off; in pmem_do_write()
188 if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) { in pmem_do_write()
209 struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data; in pmem_submit_bio()
212 if (bio->bi_opf & REQ_PREFLUSH) in pmem_submit_bio()
215 do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); in pmem_submit_bio()
226 bio->bi_status = rc; in pmem_submit_bio()
233 if (bio->bi_opf & REQ_FUA) in pmem_submit_bio()
237 bio->bi_status = errno_to_blk_status(ret); in pmem_submit_bio()
242 /* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */
247 resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset; in __pmem_direct_access()
250 struct badblocks *bb = &pmem->bb; in __pmem_direct_access()
255 *kaddr = pmem->virt_addr + offset; in __pmem_direct_access()
257 *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags); in __pmem_direct_access()
259 if (bb->count && in __pmem_direct_access()
264 return -EHWPOISON; in __pmem_direct_access()
272 PAGE_ALIGN((first_bad - sector) << SECTOR_SHIFT)); in __pmem_direct_access()
273 dev_dbg(pmem->bb.dev, "start sector(%llu), nr_pages(%ld), first_bad(%llu), actual_nr(%ld)\n", in __pmem_direct_access()
284 if (bb->count) in __pmem_direct_access()
286 return PHYS_PFN(pmem->size - pmem->pfn_pad - offset); in __pmem_direct_access()
319 * HWPoison bit, reenable page-wide read-write permission, flush the
324 * are already serialized by writer lock held by dax_iomap_rw().
332 struct device *dev = pmem->bb.dev; in pmem_recovery_write()
337 if (!is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) >> SECTOR_SHIFT, len)) in pmem_recovery_write()
341 * Not page-aligned range cannot be recovered. This should not in pmem_recovery_write()
350 pmem_off = PFN_PHYS(pgoff) + pmem->data_offset; in pmem_recovery_write()
377 struct pmem_device *pmem = dev_to_disk(dev)->private_data; in write_cache_show()
379 return sprintf(buf, "%d\n", !!dax_write_cache_enabled(pmem->dax_dev)); in write_cache_show()
385 struct pmem_device *pmem = dev_to_disk(dev)->private_data; in write_cache_store()
392 dax_write_cache(pmem->dax_dev, write_cache); in write_cache_store()
403 return a->mode; in dax_visible()
426 dax_remove_host(pmem->disk); in pmem_release_disk()
427 kill_dax(pmem->dax_dev); in pmem_release_disk()
428 put_dax(pmem->dax_dev); in pmem_release_disk()
429 del_gendisk(pmem->disk); in pmem_release_disk()
431 put_disk(pmem->disk); in pmem_release_disk()
439 u64 offset = PFN_PHYS(pfn) - pmem->phys_addr - pmem->data_offset; in pmem_pagemap_memory_failure()
442 return dax_holder_notify_failure(pmem->dax_dev, offset, len, mf_flags); in pmem_pagemap_memory_failure()
452 struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); in pmem_attach_disk()
453 struct nd_region *nd_region = to_nd_region(dev->parent); in pmem_attach_disk()
462 struct resource *res = &nsio->res; in pmem_attach_disk()
474 return -ENOMEM; in pmem_attach_disk()
483 rc = nvdimm_setup_pfn(nd_pfn, &pmem->pgmap); in pmem_attach_disk()
492 pmem->phys_addr = res->start; in pmem_attach_disk()
493 pmem->size = resource_size(res); in pmem_attach_disk()
504 if (!devm_request_mem_region(dev, res->start, resource_size(res), in pmem_attach_disk()
505 dev_name(&ndns->dev))) { in pmem_attach_disk()
507 return -EBUSY; in pmem_attach_disk()
514 pmem->disk = disk; in pmem_attach_disk()
515 pmem->pgmap.owner = pmem; in pmem_attach_disk()
516 pmem->pfn_flags = PFN_DEV; in pmem_attach_disk()
518 pmem->pgmap.type = MEMORY_DEVICE_FS_DAX; in pmem_attach_disk()
519 pmem->pgmap.ops = &fsdax_pagemap_ops; in pmem_attach_disk()
520 addr = devm_memremap_pages(dev, &pmem->pgmap); in pmem_attach_disk()
521 pfn_sb = nd_pfn->pfn_sb; in pmem_attach_disk()
522 pmem->data_offset = le64_to_cpu(pfn_sb->dataoff); in pmem_attach_disk()
523 pmem->pfn_pad = resource_size(res) - in pmem_attach_disk()
524 range_len(&pmem->pgmap.range); in pmem_attach_disk()
525 pmem->pfn_flags |= PFN_MAP; in pmem_attach_disk()
526 bb_range = pmem->pgmap.range; in pmem_attach_disk()
527 bb_range.start += pmem->data_offset; in pmem_attach_disk()
529 pmem->pgmap.range.start = res->start; in pmem_attach_disk()
530 pmem->pgmap.range.end = res->end; in pmem_attach_disk()
531 pmem->pgmap.nr_range = 1; in pmem_attach_disk()
532 pmem->pgmap.type = MEMORY_DEVICE_FS_DAX; in pmem_attach_disk()
533 pmem->pgmap.ops = &fsdax_pagemap_ops; in pmem_attach_disk()
534 addr = devm_memremap_pages(dev, &pmem->pgmap); in pmem_attach_disk()
535 pmem->pfn_flags |= PFN_MAP; in pmem_attach_disk()
536 bb_range = pmem->pgmap.range; in pmem_attach_disk()
538 addr = devm_memremap(dev, pmem->phys_addr, in pmem_attach_disk()
539 pmem->size, ARCH_MEMREMAP_PMEM); in pmem_attach_disk()
540 bb_range.start = res->start; in pmem_attach_disk()
541 bb_range.end = res->end; in pmem_attach_disk()
548 pmem->virt_addr = addr; in pmem_attach_disk()
550 disk->fops = &pmem_fops; in pmem_attach_disk()
551 disk->private_data = pmem; in pmem_attach_disk()
552 nvdimm_namespace_disk_name(ndns, disk->disk_name); in pmem_attach_disk()
553 set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset) in pmem_attach_disk()
555 if (devm_init_badblocks(dev, &pmem->bb)) in pmem_attach_disk()
556 return -ENOMEM; in pmem_attach_disk()
557 nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_range); in pmem_attach_disk()
558 disk->bb = &pmem->bb; in pmem_attach_disk()
563 if (rc != -EOPNOTSUPP) in pmem_attach_disk()
570 pmem->dax_dev = dax_dev; in pmem_attach_disk()
580 return -ENOMEM; in pmem_attach_disk()
584 pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd, in pmem_attach_disk()
586 if (!pmem->bb_state) in pmem_attach_disk()
591 dax_remove_host(pmem->disk); in pmem_attach_disk()
593 kill_dax(pmem->dax_dev); in pmem_attach_disk()
594 put_dax(pmem->dax_dev); in pmem_attach_disk()
596 put_disk(pmem->disk); in pmem_attach_disk()
621 return -ENXIO; in nd_pmem_probe()
636 return -ENXIO; in nd_pmem_probe()
637 else if (ret == -EOPNOTSUPP) in nd_pmem_probe()
642 return -ENXIO; in nd_pmem_probe()
643 else if (ret == -EOPNOTSUPP) in nd_pmem_probe()
663 sysfs_put(pmem->bb_state); in nd_pmem_remove()
664 pmem->bb_state = NULL; in nd_pmem_remove()
666 nvdimm_flush(to_nd_region(dev->parent), NULL); in nd_pmem_remove()
671 nvdimm_flush(to_nd_region(dev->parent), NULL); in nd_pmem_shutdown()
687 ndns = nd_btt->ndns; in pmem_revalidate_poison()
688 nd_region = to_nd_region(ndns->dev.parent); in pmem_revalidate_poison()
689 nsio = to_nd_namespace_io(&ndns->dev); in pmem_revalidate_poison()
690 bb = &nsio->bb; in pmem_revalidate_poison()
696 bb = &pmem->bb; in pmem_revalidate_poison()
697 bb_state = pmem->bb_state; in pmem_revalidate_poison()
701 struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb; in pmem_revalidate_poison()
703 ndns = nd_pfn->ndns; in pmem_revalidate_poison()
704 offset = pmem->data_offset + in pmem_revalidate_poison()
705 __le32_to_cpu(pfn_sb->start_pad); in pmem_revalidate_poison()
706 end_trunc = __le32_to_cpu(pfn_sb->end_trunc); in pmem_revalidate_poison()
711 nsio = to_nd_namespace_io(&ndns->dev); in pmem_revalidate_poison()
714 range.start = nsio->res.start + offset; in pmem_revalidate_poison()
715 range.end = nsio->res.end - end_trunc; in pmem_revalidate_poison()
727 struct btt *btt = nd_btt->btt; in pmem_revalidate_region()
729 nvdimm_check_and_set_ro(btt->btt_disk); in pmem_revalidate_region()
734 nvdimm_check_and_set_ro(pmem->disk); in pmem_revalidate_region()