1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/xz.h>
3 #include "compress.h"
4
5 struct z_erofs_lzma {
6 struct z_erofs_lzma *next;
7 struct xz_dec_microlzma *state;
8 u8 bounce[PAGE_SIZE];
9 };
10
11 /* considering the LZMA performance, no need to use a lockless list for now */
12 static DEFINE_SPINLOCK(z_erofs_lzma_lock);
13 static unsigned int z_erofs_lzma_max_dictsize;
14 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
15 static struct z_erofs_lzma *z_erofs_lzma_head;
16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
17
18 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
19
z_erofs_lzma_exit(void)20 static void z_erofs_lzma_exit(void)
21 {
22 /* there should be no running fs instance */
23 while (z_erofs_lzma_avail_strms) {
24 struct z_erofs_lzma *strm;
25
26 spin_lock(&z_erofs_lzma_lock);
27 strm = z_erofs_lzma_head;
28 if (!strm) {
29 spin_unlock(&z_erofs_lzma_lock);
30 DBG_BUGON(1);
31 return;
32 }
33 z_erofs_lzma_head = NULL;
34 spin_unlock(&z_erofs_lzma_lock);
35
36 while (strm) {
37 struct z_erofs_lzma *n = strm->next;
38
39 if (strm->state)
40 xz_dec_microlzma_end(strm->state);
41 kfree(strm);
42 --z_erofs_lzma_avail_strms;
43 strm = n;
44 }
45 }
46 }
47
z_erofs_lzma_init(void)48 static int __init z_erofs_lzma_init(void)
49 {
50 unsigned int i;
51
52 /* by default, use # of possible CPUs instead */
53 if (!z_erofs_lzma_nstrms)
54 z_erofs_lzma_nstrms = num_possible_cpus();
55
56 for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
57 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
58
59 if (!strm) {
60 z_erofs_lzma_exit();
61 return -ENOMEM;
62 }
63 spin_lock(&z_erofs_lzma_lock);
64 strm->next = z_erofs_lzma_head;
65 z_erofs_lzma_head = strm;
66 spin_unlock(&z_erofs_lzma_lock);
67 ++z_erofs_lzma_avail_strms;
68 }
69 return 0;
70 }
71
z_erofs_load_lzma_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)72 static int z_erofs_load_lzma_config(struct super_block *sb,
73 struct erofs_super_block *dsb, void *data, int size)
74 {
75 static DEFINE_MUTEX(lzma_resize_mutex);
76 struct z_erofs_lzma_cfgs *lzma = data;
77 unsigned int dict_size, i;
78 struct z_erofs_lzma *strm, *head = NULL;
79 int err;
80
81 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
82 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
83 return -EINVAL;
84 }
85 if (lzma->format) {
86 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
87 le16_to_cpu(lzma->format));
88 return -EINVAL;
89 }
90 dict_size = le32_to_cpu(lzma->dict_size);
91 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
92 erofs_err(sb, "unsupported lzma dictionary size %u",
93 dict_size);
94 return -EINVAL;
95 }
96
97 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
98 mutex_lock(&lzma_resize_mutex);
99
100 if (z_erofs_lzma_max_dictsize >= dict_size) {
101 mutex_unlock(&lzma_resize_mutex);
102 return 0;
103 }
104
105 /* 1. collect/isolate all streams for the following check */
106 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
107 struct z_erofs_lzma *last;
108
109 again:
110 spin_lock(&z_erofs_lzma_lock);
111 strm = z_erofs_lzma_head;
112 if (!strm) {
113 spin_unlock(&z_erofs_lzma_lock);
114 wait_event(z_erofs_lzma_wq,
115 READ_ONCE(z_erofs_lzma_head));
116 goto again;
117 }
118 z_erofs_lzma_head = NULL;
119 spin_unlock(&z_erofs_lzma_lock);
120
121 for (last = strm; last->next; last = last->next)
122 ++i;
123 last->next = head;
124 head = strm;
125 }
126
127 err = 0;
128 /* 2. walk each isolated stream and grow max dict_size if needed */
129 for (strm = head; strm; strm = strm->next) {
130 if (strm->state)
131 xz_dec_microlzma_end(strm->state);
132 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
133 if (!strm->state)
134 err = -ENOMEM;
135 }
136
137 /* 3. push back all to the global list and update max dict_size */
138 spin_lock(&z_erofs_lzma_lock);
139 DBG_BUGON(z_erofs_lzma_head);
140 z_erofs_lzma_head = head;
141 spin_unlock(&z_erofs_lzma_lock);
142 wake_up_all(&z_erofs_lzma_wq);
143
144 z_erofs_lzma_max_dictsize = dict_size;
145 mutex_unlock(&lzma_resize_mutex);
146 return err;
147 }
148
z_erofs_lzma_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)149 static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
150 struct page **pgpl)
151 {
152 struct super_block *sb = rq->sb;
153 struct z_erofs_stream_dctx dctx = {
154 .rq = rq,
155 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
156 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
157 >> PAGE_SHIFT,
158 .no = -1, .ni = 0,
159 };
160 struct xz_buf buf = {};
161 struct z_erofs_lzma *strm;
162 enum xz_ret xz_err;
163 int err;
164
165 /* 1. get the exact LZMA compressed size */
166 dctx.kin = kmap_local_page(*rq->in);
167 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
168 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
169 if (err) {
170 kunmap_local(dctx.kin);
171 return err;
172 }
173
174 /* 2. get an available lzma context */
175 again:
176 spin_lock(&z_erofs_lzma_lock);
177 strm = z_erofs_lzma_head;
178 if (!strm) {
179 spin_unlock(&z_erofs_lzma_lock);
180 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
181 goto again;
182 }
183 z_erofs_lzma_head = strm->next;
184 spin_unlock(&z_erofs_lzma_lock);
185
186 /* 3. multi-call decompress */
187 xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize,
188 !rq->partial_decoding);
189 buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
190 rq->inputsize -= buf.in_size;
191 buf.in = dctx.kin + rq->pageofs_in;
192 dctx.bounce = strm->bounce;
193 do {
194 dctx.avail_out = buf.out_size - buf.out_pos;
195 dctx.inbuf_sz = buf.in_size;
196 dctx.inbuf_pos = buf.in_pos;
197 err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out,
198 (void **)&buf.in, pgpl);
199 if (err)
200 break;
201
202 if (buf.out_size == buf.out_pos) {
203 buf.out_size = dctx.avail_out;
204 buf.out_pos = 0;
205 }
206 buf.in_size = dctx.inbuf_sz;
207 buf.in_pos = dctx.inbuf_pos;
208
209 xz_err = xz_dec_microlzma_run(strm->state, &buf);
210 DBG_BUGON(buf.out_pos > buf.out_size);
211 DBG_BUGON(buf.in_pos > buf.in_size);
212
213 if (xz_err != XZ_OK) {
214 if (xz_err == XZ_STREAM_END && !rq->outputsize)
215 break;
216 erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
217 xz_err, rq->inputsize, rq->outputsize);
218 err = -EFSCORRUPTED;
219 break;
220 }
221 } while (1);
222
223 if (dctx.kout)
224 kunmap_local(dctx.kout);
225 kunmap_local(dctx.kin);
226 /* 4. push back LZMA stream context to the global list */
227 spin_lock(&z_erofs_lzma_lock);
228 strm->next = z_erofs_lzma_head;
229 z_erofs_lzma_head = strm;
230 spin_unlock(&z_erofs_lzma_lock);
231 wake_up(&z_erofs_lzma_wq);
232 return err;
233 }
234
235 const struct z_erofs_decompressor z_erofs_lzma_decomp = {
236 .config = z_erofs_load_lzma_config,
237 .decompress = z_erofs_lzma_decompress,
238 .init = z_erofs_lzma_init,
239 .exit = z_erofs_lzma_exit,
240 .name = "lzma"
241 };
242