1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/xz.h>
3 #include "compress.h"
4 
5 struct z_erofs_lzma {
6 	struct z_erofs_lzma *next;
7 	struct xz_dec_microlzma *state;
8 	u8 bounce[PAGE_SIZE];
9 };
10 
11 /* considering the LZMA performance, no need to use a lockless list for now */
12 static DEFINE_SPINLOCK(z_erofs_lzma_lock);
13 static unsigned int z_erofs_lzma_max_dictsize;
14 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
15 static struct z_erofs_lzma *z_erofs_lzma_head;
16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
17 
18 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
19 
z_erofs_lzma_exit(void)20 static void z_erofs_lzma_exit(void)
21 {
22 	/* there should be no running fs instance */
23 	while (z_erofs_lzma_avail_strms) {
24 		struct z_erofs_lzma *strm;
25 
26 		spin_lock(&z_erofs_lzma_lock);
27 		strm = z_erofs_lzma_head;
28 		if (!strm) {
29 			spin_unlock(&z_erofs_lzma_lock);
30 			DBG_BUGON(1);
31 			return;
32 		}
33 		z_erofs_lzma_head = NULL;
34 		spin_unlock(&z_erofs_lzma_lock);
35 
36 		while (strm) {
37 			struct z_erofs_lzma *n = strm->next;
38 
39 			if (strm->state)
40 				xz_dec_microlzma_end(strm->state);
41 			kfree(strm);
42 			--z_erofs_lzma_avail_strms;
43 			strm = n;
44 		}
45 	}
46 }
47 
z_erofs_lzma_init(void)48 static int __init z_erofs_lzma_init(void)
49 {
50 	unsigned int i;
51 
52 	/* by default, use # of possible CPUs instead */
53 	if (!z_erofs_lzma_nstrms)
54 		z_erofs_lzma_nstrms = num_possible_cpus();
55 
56 	for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
57 		struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
58 
59 		if (!strm) {
60 			z_erofs_lzma_exit();
61 			return -ENOMEM;
62 		}
63 		spin_lock(&z_erofs_lzma_lock);
64 		strm->next = z_erofs_lzma_head;
65 		z_erofs_lzma_head = strm;
66 		spin_unlock(&z_erofs_lzma_lock);
67 		++z_erofs_lzma_avail_strms;
68 	}
69 	return 0;
70 }
71 
z_erofs_load_lzma_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)72 static int z_erofs_load_lzma_config(struct super_block *sb,
73 			struct erofs_super_block *dsb, void *data, int size)
74 {
75 	static DEFINE_MUTEX(lzma_resize_mutex);
76 	struct z_erofs_lzma_cfgs *lzma = data;
77 	unsigned int dict_size, i;
78 	struct z_erofs_lzma *strm, *head = NULL;
79 	int err;
80 
81 	if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
82 		erofs_err(sb, "invalid lzma cfgs, size=%u", size);
83 		return -EINVAL;
84 	}
85 	if (lzma->format) {
86 		erofs_err(sb, "unidentified lzma format %x, please check kernel version",
87 			  le16_to_cpu(lzma->format));
88 		return -EINVAL;
89 	}
90 	dict_size = le32_to_cpu(lzma->dict_size);
91 	if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
92 		erofs_err(sb, "unsupported lzma dictionary size %u",
93 			  dict_size);
94 		return -EINVAL;
95 	}
96 
97 	/* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
98 	mutex_lock(&lzma_resize_mutex);
99 
100 	if (z_erofs_lzma_max_dictsize >= dict_size) {
101 		mutex_unlock(&lzma_resize_mutex);
102 		return 0;
103 	}
104 
105 	/* 1. collect/isolate all streams for the following check */
106 	for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
107 		struct z_erofs_lzma *last;
108 
109 again:
110 		spin_lock(&z_erofs_lzma_lock);
111 		strm = z_erofs_lzma_head;
112 		if (!strm) {
113 			spin_unlock(&z_erofs_lzma_lock);
114 			wait_event(z_erofs_lzma_wq,
115 				   READ_ONCE(z_erofs_lzma_head));
116 			goto again;
117 		}
118 		z_erofs_lzma_head = NULL;
119 		spin_unlock(&z_erofs_lzma_lock);
120 
121 		for (last = strm; last->next; last = last->next)
122 			++i;
123 		last->next = head;
124 		head = strm;
125 	}
126 
127 	err = 0;
128 	/* 2. walk each isolated stream and grow max dict_size if needed */
129 	for (strm = head; strm; strm = strm->next) {
130 		if (strm->state)
131 			xz_dec_microlzma_end(strm->state);
132 		strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
133 		if (!strm->state)
134 			err = -ENOMEM;
135 	}
136 
137 	/* 3. push back all to the global list and update max dict_size */
138 	spin_lock(&z_erofs_lzma_lock);
139 	DBG_BUGON(z_erofs_lzma_head);
140 	z_erofs_lzma_head = head;
141 	spin_unlock(&z_erofs_lzma_lock);
142 	wake_up_all(&z_erofs_lzma_wq);
143 
144 	z_erofs_lzma_max_dictsize = dict_size;
145 	mutex_unlock(&lzma_resize_mutex);
146 	return err;
147 }
148 
z_erofs_lzma_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)149 static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
150 				   struct page **pgpl)
151 {
152 	struct super_block *sb = rq->sb;
153 	struct z_erofs_stream_dctx dctx = {
154 		.rq = rq,
155 		.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
156 		.outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
157 				>> PAGE_SHIFT,
158 		.no = -1, .ni = 0,
159 	};
160 	struct xz_buf buf = {};
161 	struct z_erofs_lzma *strm;
162 	enum xz_ret xz_err;
163 	int err;
164 
165 	/* 1. get the exact LZMA compressed size */
166 	dctx.kin = kmap_local_page(*rq->in);
167 	err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
168 			min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
169 	if (err) {
170 		kunmap_local(dctx.kin);
171 		return err;
172 	}
173 
174 	/* 2. get an available lzma context */
175 again:
176 	spin_lock(&z_erofs_lzma_lock);
177 	strm = z_erofs_lzma_head;
178 	if (!strm) {
179 		spin_unlock(&z_erofs_lzma_lock);
180 		wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
181 		goto again;
182 	}
183 	z_erofs_lzma_head = strm->next;
184 	spin_unlock(&z_erofs_lzma_lock);
185 
186 	/* 3. multi-call decompress */
187 	xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize,
188 			       !rq->partial_decoding);
189 	buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
190 	rq->inputsize -= buf.in_size;
191 	buf.in = dctx.kin + rq->pageofs_in;
192 	dctx.bounce = strm->bounce;
193 	do {
194 		dctx.avail_out = buf.out_size - buf.out_pos;
195 		dctx.inbuf_sz = buf.in_size;
196 		dctx.inbuf_pos = buf.in_pos;
197 		err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out,
198 						 (void **)&buf.in, pgpl);
199 		if (err)
200 			break;
201 
202 		if (buf.out_size == buf.out_pos) {
203 			buf.out_size = dctx.avail_out;
204 			buf.out_pos = 0;
205 		}
206 		buf.in_size = dctx.inbuf_sz;
207 		buf.in_pos = dctx.inbuf_pos;
208 
209 		xz_err = xz_dec_microlzma_run(strm->state, &buf);
210 		DBG_BUGON(buf.out_pos > buf.out_size);
211 		DBG_BUGON(buf.in_pos > buf.in_size);
212 
213 		if (xz_err != XZ_OK) {
214 			if (xz_err == XZ_STREAM_END && !rq->outputsize)
215 				break;
216 			erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
217 				  xz_err, rq->inputsize, rq->outputsize);
218 			err = -EFSCORRUPTED;
219 			break;
220 		}
221 	} while (1);
222 
223 	if (dctx.kout)
224 		kunmap_local(dctx.kout);
225 	kunmap_local(dctx.kin);
226 	/* 4. push back LZMA stream context to the global list */
227 	spin_lock(&z_erofs_lzma_lock);
228 	strm->next = z_erofs_lzma_head;
229 	z_erofs_lzma_head = strm;
230 	spin_unlock(&z_erofs_lzma_lock);
231 	wake_up(&z_erofs_lzma_wq);
232 	return err;
233 }
234 
235 const struct z_erofs_decompressor z_erofs_lzma_decomp = {
236 	.config = z_erofs_load_lzma_config,
237 	.decompress = z_erofs_lzma_decompress,
238 	.init = z_erofs_lzma_init,
239 	.exit = z_erofs_lzma_exit,
240 	.name = "lzma"
241 };
242