63 int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)  in dax_add_host()  argument
65 	return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL);  in dax_add_host()
85 	struct dax_device *dax_dev;  in fs_dax_get_by_bdev()  local
100 	dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);  in fs_dax_get_by_bdev()
101 	if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))  in fs_dax_get_by_bdev()
102 		dax_dev = NULL;  in fs_dax_get_by_bdev()
104 		if (!cmpxchg(&dax_dev->holder_data, NULL, holder))  in fs_dax_get_by_bdev()
105 			dax_dev->holder_ops = ops;  in fs_dax_get_by_bdev()
107 			dax_dev = NULL;  in fs_dax_get_by_bdev()
111 	return dax_dev;  in fs_dax_get_by_bdev()
115 void fs_put_dax(struct dax_device *dax_dev, void *holder)  in fs_put_dax()  argument
117 	if (dax_dev && holder &&  in fs_put_dax()
118 	    cmpxchg(&dax_dev->holder_data, holder, NULL) == holder)  in fs_put_dax()
119 		dax_dev->holder_ops = NULL;  in fs_put_dax()
120 	put_dax(dax_dev);  in fs_put_dax()
150 long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,  in dax_direct_access()  argument
155 	if (!dax_dev)  in dax_direct_access()
158 	if (!dax_alive(dax_dev))  in dax_direct_access()
164 	avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,  in dax_direct_access()
172 size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,  in dax_copy_from_iter()  argument
175 	if (!dax_alive(dax_dev))  in dax_copy_from_iter()
183 	if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))  in dax_copy_from_iter()
188 size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,  in dax_copy_to_iter()  argument
191 	if (!dax_alive(dax_dev))  in dax_copy_to_iter()
199 	if (test_bit(DAXDEV_NOMC, &dax_dev->flags))  in dax_copy_to_iter()
204 int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,  in dax_zero_page_range()  argument
209 	if (!dax_alive(dax_dev))  in dax_zero_page_range()
219 	ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);  in dax_zero_page_range()
224 size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,  in dax_recovery_write()  argument
227 	if (!dax_dev->ops->recovery_write)  in dax_recovery_write()
229 	return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);  in dax_recovery_write()
233 int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,  in dax_holder_notify_failure()  argument
239 	if (!dax_alive(dax_dev)) {  in dax_holder_notify_failure()
244 	if (!dax_dev->holder_ops) {  in dax_holder_notify_failure()
249 	rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags);  in dax_holder_notify_failure()
258 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)  in dax_flush()  argument
260 	if (unlikely(!dax_write_cache_enabled(dax_dev)))  in dax_flush()
266 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)  in dax_flush()  argument
272 void dax_write_cache(struct dax_device *dax_dev, bool wc)  in dax_write_cache()  argument
275 		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
277 		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
281 bool dax_write_cache_enabled(struct dax_device *dax_dev)  in dax_write_cache_enabled()  argument
283 	return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache_enabled()
287 bool dax_synchronous(struct dax_device *dax_dev)  in dax_synchronous()  argument
289 	return test_bit(DAXDEV_SYNC, &dax_dev->flags);  in dax_synchronous()
293 void set_dax_synchronous(struct dax_device *dax_dev)  in set_dax_synchronous()  argument
295 	set_bit(DAXDEV_SYNC, &dax_dev->flags);  in set_dax_synchronous()
299 void set_dax_nocache(struct dax_device *dax_dev)  in set_dax_nocache()  argument
301 	set_bit(DAXDEV_NOCACHE, &dax_dev->flags);  in set_dax_nocache()
305 void set_dax_nomc(struct dax_device *dax_dev)  in set_dax_nomc()  argument
307 	set_bit(DAXDEV_NOMC, &dax_dev->flags);  in set_dax_nomc()
311 bool dax_alive(struct dax_device *dax_dev)  in dax_alive()  argument
314 	return test_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_alive()
329 void kill_dax(struct dax_device *dax_dev)  in kill_dax()  argument
331 	if (!dax_dev)  in kill_dax()
334 	if (dax_dev->holder_data != NULL)  in kill_dax()
335 		dax_holder_notify_failure(dax_dev, 0, U64_MAX,  in kill_dax()
338 	clear_bit(DAXDEV_ALIVE, &dax_dev->flags);  in kill_dax()
342 	dax_dev->holder_ops = NULL;  in kill_dax()
343 	dax_dev->holder_data = NULL;  in kill_dax()
347 void run_dax(struct dax_device *dax_dev)  in run_dax()  argument
349 	set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in run_dax()
355 	struct dax_device *dax_dev;  in dax_alloc_inode()  local
358 	dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL);  in dax_alloc_inode()
359 	if (!dax_dev)  in dax_alloc_inode()
362 	inode = &dax_dev->inode;  in dax_alloc_inode()
374 	struct dax_device *dax_dev = to_dax_dev(inode);  in dax_free_inode()  local
377 	kmem_cache_free(dax_cache, dax_dev);  in dax_free_inode()
382 	struct dax_device *dax_dev = to_dax_dev(inode);  in dax_destroy_inode()  local
383 	WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),  in dax_destroy_inode()
427 	struct dax_device *dax_dev;  in dax_dev_get()  local
436 	dax_dev = to_dax_dev(inode);  in dax_dev_get()
438 		set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_dev_get()
439 		inode->i_cdev = &dax_dev->cdev;  in dax_dev_get()
446 	return dax_dev;  in dax_dev_get()
451 	struct dax_device *dax_dev;  in alloc_dax()  local
471 	dax_dev = dax_dev_get(devt);  in alloc_dax()
472 	if (!dax_dev)  in alloc_dax()
475 	dax_dev->ops = ops;  in alloc_dax()
476 	dax_dev->private = private;  in alloc_dax()
477 	return dax_dev;  in alloc_dax()
485 void put_dax(struct dax_device *dax_dev)  in put_dax()  argument
487 	if (!dax_dev)  in put_dax()
489 	iput(&dax_dev->inode);  in put_dax()
500 void *dax_holder(struct dax_device *dax_dev)  in dax_holder()  argument
502 	return dax_dev->holder_data;  in dax_holder()
521 struct inode *dax_inode(struct dax_device *dax_dev)  in dax_inode()  argument
523 	return &dax_dev->inode;  in dax_inode()
527 void *dax_get_private(struct dax_device *dax_dev)  in dax_get_private()  argument
529 	if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))  in dax_get_private()
531 	return dax_dev->private;  in dax_get_private()
537 	struct dax_device *dax_dev = _dax_dev;  in init_once()  local
538 	struct inode *inode = &dax_dev->inode;  in init_once()
540 	memset(dax_dev, 0, sizeof(*dax_dev));  in init_once()