1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * zswap.c - zswap driver file
7  * RAM-based memory pool.  This can result in a significant I/O reduction on
32 #include <linux/page-flags.h>
125 /* Enable/disable memory pressure-based shrinker. */
154  * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock.
157  * needs to be verified that it's still valid in the tree.
184  * swpentry - associated swap entry, the offset indexes into the red-black tree
185  * length - the length in bytes of the compressed page data.  Needed during
187  * referenced - true if the entry recently entered the zswap pool. Unset by the
191  * pool - the zswap_pool the entry's data is in
192  * handle - zpool allocation handle that stores the compressed page data
193  * objcg - the obj_cgroup that the compressed memory is charged to
194  * lru - handle to the pool's lru used to evict pages.
209 /* RCU-protected iteration */
241 	pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name,		\
242 		 zpool_get_type((p)->zpool))
273 	pool->zpool = zpool_create_pool(type, name, gfp);  in zswap_pool_create()
274 	if (!pool->zpool) {  in zswap_pool_create()
278 	pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));  in zswap_pool_create()
280 	strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));  in zswap_pool_create()
282 	pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx);  in zswap_pool_create()
283 	if (!pool->acomp_ctx) {  in zswap_pool_create()
289 				       &pool->node);  in zswap_pool_create()
296 	ret = percpu_ref_init(&pool->ref, __zswap_pool_empty,  in zswap_pool_create()
300 	INIT_LIST_HEAD(&pool->list);  in zswap_pool_create()
307 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);  in zswap_pool_create()
309 	if (pool->acomp_ctx)  in zswap_pool_create()
310 		free_percpu(pool->acomp_ctx);  in zswap_pool_create()
311 	if (pool->zpool)  in zswap_pool_create()
312 		zpool_destroy_pool(pool->zpool);  in zswap_pool_create()
363 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);  in zswap_pool_destroy()
364 	free_percpu(pool->acomp_ctx);  in zswap_pool_destroy()
366 	zpool_destroy_pool(pool->zpool);  in zswap_pool_destroy()
378 	WARN_ON(!percpu_ref_is_zero(&pool->ref));  in __zswap_pool_release()
379 	percpu_ref_exit(&pool->ref);  in __zswap_pool_release()
397 	list_del_rcu(&pool->list);  in __zswap_pool_empty()
399 	INIT_WORK(&pool->release_work, __zswap_pool_release);  in __zswap_pool_empty()
400 	schedule_work(&pool->release_work);  in __zswap_pool_empty()
410 	return percpu_ref_tryget(&pool->ref);  in zswap_pool_get()
415 	percpu_ref_put(&pool->ref);  in zswap_pool_put()
451 /* type and compressor must be null-terminated */
459 		if (strcmp(pool->tfm_name, compressor))  in zswap_pool_find_get()
461 		if (strcmp(zpool_get_type(pool->zpool), type))  in zswap_pool_find_get()
489 		total += zpool_get_total_pages(pool->zpool);  in zswap_total_pages()
517 	if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)  in zswap_pool_changed()
522 /* val must be a null-terminated string */
534 		/* if this is load-time (pre-init) param setting,  in __zswap_param_set()
544 		ret = -ENODEV;  in __zswap_param_set()
555 			return -ENOENT;  in __zswap_param_set()
561 			return -ENOENT;  in __zswap_param_set()
566 		return -EINVAL;  in __zswap_param_set()
575 		list_del_rcu(&pool->list);  in __zswap_param_set()
588 		percpu_ref_resurrect(&pool->ref);  in __zswap_param_set()
597 		ret = -EINVAL;  in __zswap_param_set()
603 		list_add_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
606 		/* add the possibly pre-existing pool to the end of the pools  in __zswap_param_set()
610 		list_add_tail_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
620 		 * when the other param is changed. We already verified this  in __zswap_param_set()
631 		percpu_ref_kill(&put_pool->ref);  in __zswap_param_set()
651 	int ret = -ENODEV;  in zswap_enabled_param_set()
653 	/* if this is load-time (pre-init) param setting, only set param. */  in zswap_enabled_param_set()
685 	return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL;  in mem_cgroup_from_entry()
706 	 * concurrent memcg offlining. Thanks to the memcg->kmemcg_id indirection  in zswap_lru_add()
709 	 * 1. list_lru_add() is called before memcg->kmemcg_id is updated. The  in zswap_lru_add()
711 	 * 2. list_lru_add() is called after memcg->kmemcg_id is updated. The  in zswap_lru_add()
719 	list_lru_add(list_lru, &entry->lru, nid, memcg);  in zswap_lru_add()
731 	list_lru_del(list_lru, &entry->lru, nid, memcg);  in zswap_lru_del()
737 	atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0);  in zswap_lruvec_state_init()
746 		atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins);  in zswap_folio_swapin()
798 	zpool_free(entry->pool->zpool, entry->handle);  in zswap_entry_free()
799 	zswap_pool_put(entry->pool);  in zswap_entry_free()
800 	if (entry->objcg) {  in zswap_entry_free()
801 		obj_cgroup_uncharge_zswap(entry->objcg, entry->length);  in zswap_entry_free()
802 		obj_cgroup_put(entry->objcg);  in zswap_entry_free()
814 	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);  in zswap_cpu_comp_prepare()
819 	mutex_init(&acomp_ctx->mutex);  in zswap_cpu_comp_prepare()
821 	acomp_ctx->buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));  in zswap_cpu_comp_prepare()
822 	if (!acomp_ctx->buffer)  in zswap_cpu_comp_prepare()
823 		return -ENOMEM;  in zswap_cpu_comp_prepare()
825 	acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));  in zswap_cpu_comp_prepare()
828 				pool->tfm_name, PTR_ERR(acomp));  in zswap_cpu_comp_prepare()
832 	acomp_ctx->acomp = acomp;  in zswap_cpu_comp_prepare()
833 	acomp_ctx->is_sleepable = acomp_is_async(acomp);  in zswap_cpu_comp_prepare()
835 	req = acomp_request_alloc(acomp_ctx->acomp);  in zswap_cpu_comp_prepare()
838 		       pool->tfm_name);  in zswap_cpu_comp_prepare()
839 		ret = -ENOMEM;  in zswap_cpu_comp_prepare()
842 	acomp_ctx->req = req;  in zswap_cpu_comp_prepare()
844 	crypto_init_wait(&acomp_ctx->wait);  in zswap_cpu_comp_prepare()
851 				   crypto_req_done, &acomp_ctx->wait);  in zswap_cpu_comp_prepare()
856 	crypto_free_acomp(acomp_ctx->acomp);  in zswap_cpu_comp_prepare()
858 	kfree(acomp_ctx->buffer);  in zswap_cpu_comp_prepare()
865 	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);  in zswap_cpu_comp_dead()
868 		if (!IS_ERR_OR_NULL(acomp_ctx->req))  in zswap_cpu_comp_dead()
869 			acomp_request_free(acomp_ctx->req);  in zswap_cpu_comp_dead()
870 		if (!IS_ERR_OR_NULL(acomp_ctx->acomp))  in zswap_cpu_comp_dead()
871 			crypto_free_acomp(acomp_ctx->acomp);  in zswap_cpu_comp_dead()
872 		kfree(acomp_ctx->buffer);  in zswap_cpu_comp_dead()
890 	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);  in zswap_compress()
892 	mutex_lock(&acomp_ctx->mutex);  in zswap_compress()
894 	dst = acomp_ctx->buffer;  in zswap_compress()
899 	 * We need PAGE_SIZE * 2 here since there maybe over-compression case,  in zswap_compress()
900 	 * and hardware-accelerators may won't check the dst buffer size, so  in zswap_compress()
904 	acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);  in zswap_compress()
918 	comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);  in zswap_compress()
919 	dlen = acomp_ctx->req->dlen;  in zswap_compress()
923 	zpool = entry->pool->zpool;  in zswap_compress()
935 	entry->handle = handle;  in zswap_compress()
936 	entry->length = dlen;  in zswap_compress()
939 	if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC)  in zswap_compress()
946 	mutex_unlock(&acomp_ctx->mutex);  in zswap_compress()
952 	struct zpool *zpool = entry->pool->zpool;  in zswap_decompress()
957 	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);  in zswap_decompress()
958 	mutex_lock(&acomp_ctx->mutex);  in zswap_decompress()
960 	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);  in zswap_decompress()
965 	 * Meanwhile, zpool_map_handle() might return a non-linearly mapped buffer,  in zswap_decompress()
970 	if ((acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) ||  in zswap_decompress()
972 		memcpy(acomp_ctx->buffer, src, entry->length);  in zswap_decompress()
973 		src = acomp_ctx->buffer;  in zswap_decompress()
974 		zpool_unmap_handle(zpool, entry->handle);  in zswap_decompress()
977 	sg_init_one(&input, src, entry->length);  in zswap_decompress()
980 	acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);  in zswap_decompress()
981 	BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));  in zswap_decompress()
982 	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);  in zswap_decompress()
983 	mutex_unlock(&acomp_ctx->mutex);  in zswap_decompress()
985 	if (src != acomp_ctx->buffer)  in zswap_decompress()
986 		zpool_unmap_handle(zpool, entry->handle);  in zswap_decompress()
1021 		return -ENOMEM;  in zswap_writeback_entry()
1032 		return -EEXIST;  in zswap_writeback_entry()
1049 		return -ENOMEM;  in zswap_writeback_entry()
1055 	if (entry->objcg)  in zswap_writeback_entry()
1056 		count_objcg_events(entry->objcg, ZSWPWB, 1);  in zswap_writeback_entry()
1083  *    adjusted by the pool activities - if the pool is dominated by new entries
1112 	if (entry->referenced) {  in shrink_memcg_cb()
1113 		entry->referenced = false;  in shrink_memcg_cb()
1138 	 *    We don't do any trylocking; -ENOMEM comes closest,  in shrink_memcg_cb()
1142 	list_move_tail(item, &l->list);  in shrink_memcg_cb()
1147 	 * until the entry is verified to still be alive in the tree.  in shrink_memcg_cb()
1149 	swpentry = entry->swpentry;  in shrink_memcg_cb()
1168 		if (writeback_result == -EEXIST && encountered_page_in_swapcache) {  in shrink_memcg_cb()
1187 			!mem_cgroup_zswap_writeback_enabled(sc->memcg)) {  in zswap_shrinker_scan()
1188 		sc->nr_scanned = 0;  in zswap_shrinker_scan()
1204 	struct mem_cgroup *memcg = sc->memcg;  in zswap_shrinker_count()
1205 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid));  in zswap_shrinker_count()
1207 		&lruvec->zswap_lruvec_state.nr_disk_swapins;  in zswap_shrinker_count()
1217 	 * rules (may_enter_fs()), which apply on a per-folio basis.  in zswap_shrinker_count()
1219 	if (!gfp_has_io_fs(sc->gfp_mask))  in zswap_shrinker_count()
1223 	 * For memcg, use the cgroup-wide ZSWAP stats since we don't  in zswap_shrinker_count()
1224 	 * have them per-node and thus per-lruvec. Careful if memcg is  in zswap_shrinker_count()
1225 	 * runtime-disabled: we can get sc->memcg == NULL, which is ok  in zswap_shrinker_count()
1228 	 * Without memcg, use the zswap pool-wide metrics.  in zswap_shrinker_count()
1256 			nr_remain = nr_disk_swapins_cur - nr_freeable;  in zswap_shrinker_count()
1260 	nr_freeable -= nr_disk_swapins_cur - nr_remain;  in zswap_shrinker_count()
1278 		shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap");  in zswap_alloc_shrinker()
1282 	shrinker->scan_objects = zswap_shrinker_scan;  in zswap_alloc_shrinker()
1283 	shrinker->count_objects = zswap_shrinker_count;  in zswap_alloc_shrinker()
1284 	shrinker->batch = 0;  in zswap_alloc_shrinker()
1285 	shrinker->seeks = DEFAULT_SEEKS;  in zswap_alloc_shrinker()
1294 		return -ENOENT;  in shrink_memcg()
1301 		return -ENOENT;  in shrink_memcg()
1308 		scanned += 1 - nr_to_walk;  in shrink_memcg()
1312 		return -ENOENT;  in shrink_memcg()
1314 	return shrunk ? 0 : -EAGAIN;  in shrink_memcg()
1327 	 * Global reclaim will select cgroup in a round-robin fashion from all  in shrink_worker()
1329 	 * writeback-disabled memcgs (memory.zswap.writeback=0) are not  in shrink_worker()
1334 	 * - No writeback-candidate memcgs found in a memcg tree walk.  in shrink_worker()
1335 	 * - Shrinking a writeback-candidate memcg failed.  in shrink_worker()
1387 		 * There are no writeback-candidate pages in the memcg.  in shrink_worker()
1392 		if (ret == -ENOENT)  in shrink_worker()
1408 	swp_entry_t swp = folio->swap;  in zswap_store()
1447 	entry->pool = zswap_pool_current_get();  in zswap_store()
1448 	if (!entry->pool)  in zswap_store()
1463 	entry->swpentry = swp;  in zswap_store()
1464 	entry->objcg = objcg;  in zswap_store()
1465 	entry->referenced = true;  in zswap_store()
1471 		WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);  in zswap_store()
1485 		obj_cgroup_charge_zswap(objcg, entry->length);  in zswap_store()
1499 	if (entry->length) {  in zswap_store()
1500 		INIT_LIST_HEAD(&entry->lru);  in zswap_store()
1511 	zpool_free(entry->pool->zpool, entry->handle);  in zswap_store()
1513 	zswap_pool_put(entry->pool);  in zswap_store()
1534 	swp_entry_t swp = folio->swap;  in zswap_load()
1560 	 * in-memory copies outweighs any benefits of caching the  in zswap_load()
1579 	if (entry->objcg)  in zswap_load()
1580 		count_objcg_events(entry->objcg, ZSWPIN, 1);  in zswap_load()
1611 		return -ENOMEM;  in zswap_swapon()
1657 		return -ENODEV;  in zswap_debugfs_init()
1710 	shrink_wq = alloc_workqueue("zswap-shrink",  in zswap_setup()
1726 		pr_info("loaded using pool %s/%s\n", pool->tfm_name,  in zswap_setup()
1727 			zpool_get_type(pool->zpool));  in zswap_setup()
1728 		list_add(&pool->list, &zswap_pools);  in zswap_setup()
1750 	/* if built-in, we aren't unloaded on failure; don't allow use */  in zswap_setup()
1753 	return -ENOMEM;  in zswap_setup()