35 static struct device *to_dev(struct pmem_device *pmem)  in to_dev()  argument
41 	return pmem->bb.dev;  in to_dev()
44 static struct nd_region *to_region(struct pmem_device *pmem)  in to_region()  argument
46 	return to_nd_region(to_dev(pmem)->parent);  in to_region()
49 static phys_addr_t pmem_to_phys(struct pmem_device *pmem, phys_addr_t offset)  in pmem_to_phys()  argument
51 	return pmem->phys_addr + offset;  in pmem_to_phys()
54 static sector_t to_sect(struct pmem_device *pmem, phys_addr_t offset)  in to_sect()  argument
56 	return (offset - pmem->data_offset) >> SECTOR_SHIFT;  in to_sect()
59 static phys_addr_t to_offset(struct pmem_device *pmem, sector_t sector)  in to_offset()  argument
61 	return (sector << SECTOR_SHIFT) + pmem->data_offset;  in to_offset()
64 static void pmem_mkpage_present(struct pmem_device *pmem, phys_addr_t offset,  in pmem_mkpage_present()  argument
67 	phys_addr_t phys = pmem_to_phys(pmem, offset);  in pmem_mkpage_present()
71 	if (is_vmalloc_addr(pmem->virt_addr))  in pmem_mkpage_present()
89 static void pmem_clear_bb(struct pmem_device *pmem, sector_t sector, long blks)  in pmem_clear_bb()  argument
93 	badblocks_clear(&pmem->bb, sector, blks);  in pmem_clear_bb()
94 	if (pmem->bb_state)  in pmem_clear_bb()
95 		sysfs_notify_dirent(pmem->bb_state);  in pmem_clear_bb()
98 static long __pmem_clear_poison(struct pmem_device *pmem,  in __pmem_clear_poison()  argument
101 	phys_addr_t phys = pmem_to_phys(pmem, offset);  in __pmem_clear_poison()
102 	long cleared = nvdimm_clear_poison(to_dev(pmem), phys, len);  in __pmem_clear_poison()
105 		pmem_mkpage_present(pmem, offset, cleared);  in __pmem_clear_poison()
106 		arch_invalidate_pmem(pmem->virt_addr + offset, len);  in __pmem_clear_poison()
111 static blk_status_t pmem_clear_poison(struct pmem_device *pmem,  in pmem_clear_poison()  argument
114 	long cleared = __pmem_clear_poison(pmem, offset, len);  in pmem_clear_poison()
119 	pmem_clear_bb(pmem, to_sect(pmem, offset), cleared >> SECTOR_SHIFT);  in pmem_clear_poison()
165 static blk_status_t pmem_do_read(struct pmem_device *pmem,  in pmem_do_read()  argument
170 	phys_addr_t pmem_off = to_offset(pmem, sector);  in pmem_do_read()
171 	void *pmem_addr = pmem->virt_addr + pmem_off;  in pmem_do_read()
173 	if (unlikely(is_bad_pmem(&pmem->bb, sector, len)))  in pmem_do_read()
181 static blk_status_t pmem_do_write(struct pmem_device *pmem,  in pmem_do_write()  argument
185 	phys_addr_t pmem_off = to_offset(pmem, sector);  in pmem_do_write()
186 	void *pmem_addr = pmem->virt_addr + pmem_off;  in pmem_do_write()
188 	if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) {  in pmem_do_write()
189 		blk_status_t rc = pmem_clear_poison(pmem, pmem_off, len);  in pmem_do_write()
209 	struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data;  in pmem_submit_bio()  local
210 	struct nd_region *nd_region = to_region(pmem);  in pmem_submit_bio()
220 			rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset,  in pmem_submit_bio()
223 			rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset,  in pmem_submit_bio()
243 __weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,  in __pmem_direct_access()  argument
247 	resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset;  in __pmem_direct_access()
250 	struct badblocks *bb = &pmem->bb;  in __pmem_direct_access()
255 		*kaddr = pmem->virt_addr + offset;  in __pmem_direct_access()
257 		*pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);  in __pmem_direct_access()
273 		dev_dbg(pmem->bb.dev, "start sector(%llu), nr_pages(%ld), first_bad(%llu), actual_nr(%ld)\n",  in __pmem_direct_access()
286 	return PHYS_PFN(pmem->size - pmem->pfn_pad - offset);  in __pmem_direct_access()
297 	struct pmem_device *pmem = dax_get_private(dax_dev);  in pmem_dax_zero_page_range()  local
299 	return blk_status_to_errno(pmem_do_write(pmem, ZERO_PAGE(0), 0,  in pmem_dax_zero_page_range()
308 	struct pmem_device *pmem = dax_get_private(dax_dev);  in pmem_dax_direct_access()  local
310 	return __pmem_direct_access(pmem, pgoff, nr_pages, mode, kaddr, pfn);  in pmem_dax_direct_access()
329 	struct pmem_device *pmem = dax_get_private(dax_dev);  in pmem_recovery_write()  local
332 	struct device *dev = pmem->bb.dev;  in pmem_recovery_write()
337 	if (!is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) >> SECTOR_SHIFT, len))  in pmem_recovery_write()
350 	pmem_off = PFN_PHYS(pgoff) + pmem->data_offset;  in pmem_recovery_write()
351 	cleared = __pmem_clear_poison(pmem, pmem_off, len);  in pmem_recovery_write()
363 	pmem_clear_bb(pmem, to_sect(pmem, pmem_off), cleared >> SECTOR_SHIFT);  in pmem_recovery_write()
377 	struct pmem_device *pmem = dev_to_disk(dev)->private_data;  in write_cache_show()  local
379 	return sprintf(buf, "%d\n", !!dax_write_cache_enabled(pmem->dax_dev));  in write_cache_show()
385 	struct pmem_device *pmem = dev_to_disk(dev)->private_data;  in write_cache_store()  local
392 	dax_write_cache(pmem->dax_dev, write_cache);  in write_cache_store()
424 	struct pmem_device *pmem = __pmem;  in pmem_release_disk()  local
426 	dax_remove_host(pmem->disk);  in pmem_release_disk()
427 	kill_dax(pmem->dax_dev);  in pmem_release_disk()
428 	put_dax(pmem->dax_dev);  in pmem_release_disk()
429 	del_gendisk(pmem->disk);  in pmem_release_disk()
431 	put_disk(pmem->disk);  in pmem_release_disk()
437 	struct pmem_device *pmem =  in pmem_pagemap_memory_failure()  local
439 	u64 offset = PFN_PHYS(pfn) - pmem->phys_addr - pmem->data_offset;  in pmem_pagemap_memory_failure()
442 	return dax_holder_notify_failure(pmem->dax_dev, offset, len, mf_flags);  in pmem_pagemap_memory_failure()
467 	struct pmem_device *pmem;  in pmem_attach_disk()  local
472 	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);  in pmem_attach_disk()
473 	if (!pmem)  in pmem_attach_disk()
483 		rc = nvdimm_setup_pfn(nd_pfn, &pmem->pgmap);  in pmem_attach_disk()
491 	dev_set_drvdata(dev, pmem);  in pmem_attach_disk()
492 	pmem->phys_addr = res->start;  in pmem_attach_disk()
493 	pmem->size = resource_size(res);  in pmem_attach_disk()
514 	pmem->disk = disk;  in pmem_attach_disk()
515 	pmem->pgmap.owner = pmem;  in pmem_attach_disk()
516 	pmem->pfn_flags = PFN_DEV;  in pmem_attach_disk()
518 		pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;  in pmem_attach_disk()
519 		pmem->pgmap.ops = &fsdax_pagemap_ops;  in pmem_attach_disk()
520 		addr = devm_memremap_pages(dev, &pmem->pgmap);  in pmem_attach_disk()
522 		pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);  in pmem_attach_disk()
523 		pmem->pfn_pad = resource_size(res) -  in pmem_attach_disk()
524 			range_len(&pmem->pgmap.range);  in pmem_attach_disk()
525 		pmem->pfn_flags |= PFN_MAP;  in pmem_attach_disk()
526 		bb_range = pmem->pgmap.range;  in pmem_attach_disk()
527 		bb_range.start += pmem->data_offset;  in pmem_attach_disk()
529 		pmem->pgmap.range.start = res->start;  in pmem_attach_disk()
530 		pmem->pgmap.range.end = res->end;  in pmem_attach_disk()
531 		pmem->pgmap.nr_range = 1;  in pmem_attach_disk()
532 		pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;  in pmem_attach_disk()
533 		pmem->pgmap.ops = &fsdax_pagemap_ops;  in pmem_attach_disk()
534 		addr = devm_memremap_pages(dev, &pmem->pgmap);  in pmem_attach_disk()
535 		pmem->pfn_flags |= PFN_MAP;  in pmem_attach_disk()
536 		bb_range = pmem->pgmap.range;  in pmem_attach_disk()
538 		addr = devm_memremap(dev, pmem->phys_addr,  in pmem_attach_disk()
539 				pmem->size, ARCH_MEMREMAP_PMEM);  in pmem_attach_disk()
548 	pmem->virt_addr = addr;  in pmem_attach_disk()
551 	disk->private_data	= pmem;  in pmem_attach_disk()
553 	set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)  in pmem_attach_disk()
555 	if (devm_init_badblocks(dev, &pmem->bb))  in pmem_attach_disk()
557 	nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_range);  in pmem_attach_disk()
558 	disk->bb = &pmem->bb;  in pmem_attach_disk()
560 	dax_dev = alloc_dax(pmem, &pmem_dax_ops);  in pmem_attach_disk()
570 		pmem->dax_dev = dax_dev;  in pmem_attach_disk()
579 	if (devm_add_action_or_reset(dev, pmem_release_disk, pmem))  in pmem_attach_disk()
584 	pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd,  in pmem_attach_disk()
586 	if (!pmem->bb_state)  in pmem_attach_disk()
591 	dax_remove_host(pmem->disk);  in pmem_attach_disk()
593 	kill_dax(pmem->dax_dev);  in pmem_attach_disk()
594 	put_dax(pmem->dax_dev);  in pmem_attach_disk()
596 	put_disk(pmem->disk);  in pmem_attach_disk()
654 	struct pmem_device *pmem = dev_get_drvdata(dev);  in nd_pmem_remove()  local
663 		sysfs_put(pmem->bb_state);  in nd_pmem_remove()
664 		pmem->bb_state = NULL;  in nd_pmem_remove()
693 		struct pmem_device *pmem = dev_get_drvdata(dev);  in pmem_revalidate_poison()  local
695 		nd_region = to_region(pmem);  in pmem_revalidate_poison()
696 		bb = &pmem->bb;  in pmem_revalidate_poison()
697 		bb_state = pmem->bb_state;  in pmem_revalidate_poison()
704 			offset = pmem->data_offset +  in pmem_revalidate_poison()
723 	struct pmem_device *pmem;  in pmem_revalidate_region()  local
733 	pmem = dev_get_drvdata(dev);  in pmem_revalidate_region()
734 	nvdimm_check_and_set_ro(pmem->disk);  in pmem_revalidate_region()