1  // SPDX-License-Identifier: GPL-2.0-or-later
2  #include <linux/zstd.h>
3  #include "compress.h"
4  
5  struct z_erofs_zstd {
6  	struct z_erofs_zstd *next;
7  	u8 bounce[PAGE_SIZE];
8  	void *wksp;
9  	unsigned int wkspsz;
10  };
11  
12  static DEFINE_SPINLOCK(z_erofs_zstd_lock);
13  static unsigned int z_erofs_zstd_max_dictsize;
14  static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms;
15  static struct z_erofs_zstd *z_erofs_zstd_head;
16  static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq);
17  
18  module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444);
19  
z_erofs_isolate_strms(bool all)20  static struct z_erofs_zstd *z_erofs_isolate_strms(bool all)
21  {
22  	struct z_erofs_zstd *strm;
23  
24  again:
25  	spin_lock(&z_erofs_zstd_lock);
26  	strm = z_erofs_zstd_head;
27  	if (!strm) {
28  		spin_unlock(&z_erofs_zstd_lock);
29  		wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head));
30  		goto again;
31  	}
32  	z_erofs_zstd_head = all ? NULL : strm->next;
33  	spin_unlock(&z_erofs_zstd_lock);
34  	return strm;
35  }
36  
z_erofs_zstd_exit(void)37  static void z_erofs_zstd_exit(void)
38  {
39  	while (z_erofs_zstd_avail_strms) {
40  		struct z_erofs_zstd *strm, *n;
41  
42  		for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
43  			n = strm->next;
44  
45  			kvfree(strm->wksp);
46  			kfree(strm);
47  			--z_erofs_zstd_avail_strms;
48  		}
49  	}
50  }
51  
z_erofs_zstd_init(void)52  static int __init z_erofs_zstd_init(void)
53  {
54  	/* by default, use # of possible CPUs instead */
55  	if (!z_erofs_zstd_nstrms)
56  		z_erofs_zstd_nstrms = num_possible_cpus();
57  
58  	for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms;
59  	     ++z_erofs_zstd_avail_strms) {
60  		struct z_erofs_zstd *strm;
61  
62  		strm = kzalloc(sizeof(*strm), GFP_KERNEL);
63  		if (!strm) {
64  			z_erofs_zstd_exit();
65  			return -ENOMEM;
66  		}
67  		spin_lock(&z_erofs_zstd_lock);
68  		strm->next = z_erofs_zstd_head;
69  		z_erofs_zstd_head = strm;
70  		spin_unlock(&z_erofs_zstd_lock);
71  	}
72  	return 0;
73  }
74  
z_erofs_load_zstd_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)75  static int z_erofs_load_zstd_config(struct super_block *sb,
76  			struct erofs_super_block *dsb, void *data, int size)
77  {
78  	static DEFINE_MUTEX(zstd_resize_mutex);
79  	struct z_erofs_zstd_cfgs *zstd = data;
80  	unsigned int dict_size, wkspsz;
81  	struct z_erofs_zstd *strm, *head = NULL;
82  	void *wksp;
83  
84  	if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) {
85  		erofs_err(sb, "unsupported zstd format, size=%u", size);
86  		return -EINVAL;
87  	}
88  
89  	if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) {
90  		erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog);
91  		return -EINVAL;
92  	}
93  	dict_size = 1U << (zstd->windowlog + 10);
94  
95  	/* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */
96  	mutex_lock(&zstd_resize_mutex);
97  	if (z_erofs_zstd_max_dictsize >= dict_size) {
98  		mutex_unlock(&zstd_resize_mutex);
99  		return 0;
100  	}
101  
102  	/* 1. collect/isolate all streams for the following check */
103  	while (z_erofs_zstd_avail_strms) {
104  		struct z_erofs_zstd *n;
105  
106  		for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
107  			n = strm->next;
108  			strm->next = head;
109  			head = strm;
110  			--z_erofs_zstd_avail_strms;
111  		}
112  	}
113  
114  	/* 2. walk each isolated stream and grow max dict_size if needed */
115  	wkspsz = zstd_dstream_workspace_bound(dict_size);
116  	for (strm = head; strm; strm = strm->next) {
117  		wksp = kvmalloc(wkspsz, GFP_KERNEL);
118  		if (!wksp)
119  			break;
120  		kvfree(strm->wksp);
121  		strm->wksp = wksp;
122  		strm->wkspsz = wkspsz;
123  	}
124  
125  	/* 3. push back all to the global list and update max dict_size */
126  	spin_lock(&z_erofs_zstd_lock);
127  	DBG_BUGON(z_erofs_zstd_head);
128  	z_erofs_zstd_head = head;
129  	spin_unlock(&z_erofs_zstd_lock);
130  	z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms;
131  	wake_up_all(&z_erofs_zstd_wq);
132  	if (!strm)
133  		z_erofs_zstd_max_dictsize = dict_size;
134  	mutex_unlock(&zstd_resize_mutex);
135  	return strm ? -ENOMEM : 0;
136  }
137  
z_erofs_zstd_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)138  static int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq,
139  				   struct page **pgpl)
140  {
141  	struct super_block *sb = rq->sb;
142  	struct z_erofs_stream_dctx dctx = {
143  		.rq = rq,
144  		.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
145  		.outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
146  				>> PAGE_SHIFT,
147  		.no = -1, .ni = 0,
148  	};
149  	zstd_in_buffer in_buf = { NULL, 0, 0 };
150  	zstd_out_buffer out_buf = { NULL, 0, 0 };
151  	struct z_erofs_zstd *strm;
152  	zstd_dstream *stream;
153  	int zerr, err;
154  
155  	/* 1. get the exact compressed size */
156  	dctx.kin = kmap_local_page(*rq->in);
157  	err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
158  			min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
159  	if (err) {
160  		kunmap_local(dctx.kin);
161  		return err;
162  	}
163  
164  	/* 2. get an available ZSTD context */
165  	strm = z_erofs_isolate_strms(false);
166  
167  	/* 3. multi-call decompress */
168  	stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz);
169  	if (!stream) {
170  		err = -EIO;
171  		goto failed_zinit;
172  	}
173  
174  	rq->fillgaps = true;	/* ZSTD doesn't support NULL output buffer */
175  	in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in);
176  	rq->inputsize -= in_buf.size;
177  	in_buf.src = dctx.kin + rq->pageofs_in;
178  	dctx.bounce = strm->bounce;
179  
180  	do {
181  		dctx.avail_out = out_buf.size - out_buf.pos;
182  		dctx.inbuf_sz = in_buf.size;
183  		dctx.inbuf_pos = in_buf.pos;
184  		err = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst,
185  						 (void **)&in_buf.src, pgpl);
186  		if (err)
187  			break;
188  
189  		if (out_buf.size == out_buf.pos) {
190  			out_buf.size = dctx.avail_out;
191  			out_buf.pos = 0;
192  		}
193  		in_buf.size = dctx.inbuf_sz;
194  		in_buf.pos = dctx.inbuf_pos;
195  
196  		zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);
197  		if (zstd_is_error(zerr) || (!zerr && rq->outputsize)) {
198  			erofs_err(sb, "failed to decompress in[%u] out[%u]: %s",
199  				  rq->inputsize, rq->outputsize,
200  				  zerr ? zstd_get_error_name(zerr) : "unexpected end of stream");
201  			err = -EFSCORRUPTED;
202  			break;
203  		}
204  	} while (rq->outputsize || out_buf.pos < out_buf.size);
205  
206  	if (dctx.kout)
207  		kunmap_local(dctx.kout);
208  failed_zinit:
209  	kunmap_local(dctx.kin);
210  	/* 4. push back ZSTD stream context to the global list */
211  	spin_lock(&z_erofs_zstd_lock);
212  	strm->next = z_erofs_zstd_head;
213  	z_erofs_zstd_head = strm;
214  	spin_unlock(&z_erofs_zstd_lock);
215  	wake_up(&z_erofs_zstd_wq);
216  	return err;
217  }
218  
219  const struct z_erofs_decompressor z_erofs_zstd_decomp = {
220  	.config = z_erofs_load_zstd_config,
221  	.decompress = z_erofs_zstd_decompress,
222  	.init = z_erofs_zstd_init,
223  	.exit = z_erofs_zstd_exit,
224  	.name = "zstd",
225  };
226