1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux
4 */
5
6 #include <linux/crc32.h>
7 #include <linux/base64.h>
8 #include <linux/prandom.h>
9 #include <linux/unaligned.h>
10 #include <crypto/hash.h>
11 #include <crypto/dh.h>
12 #include "nvme.h"
13 #include "fabrics.h"
14 #include <linux/nvme-auth.h>
15
16 #define CHAP_BUF_SIZE 4096
17 static struct kmem_cache *nvme_chap_buf_cache;
18 static mempool_t *nvme_chap_buf_pool;
19
20 struct nvme_dhchap_queue_context {
21 struct list_head entry;
22 struct work_struct auth_work;
23 struct nvme_ctrl *ctrl;
24 struct crypto_shash *shash_tfm;
25 struct crypto_kpp *dh_tfm;
26 struct nvme_dhchap_key *transformed_key;
27 void *buf;
28 int qid;
29 int error;
30 u32 s1;
31 u32 s2;
32 bool bi_directional;
33 u16 transaction;
34 u8 status;
35 u8 dhgroup_id;
36 u8 hash_id;
37 size_t hash_len;
38 u8 c1[64];
39 u8 c2[64];
40 u8 response[64];
41 u8 *ctrl_key;
42 u8 *host_key;
43 u8 *sess_key;
44 int ctrl_key_len;
45 int host_key_len;
46 int sess_key_len;
47 };
48
49 static struct workqueue_struct *nvme_auth_wq;
50
ctrl_max_dhchaps(struct nvme_ctrl * ctrl)51 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)
52 {
53 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +
54 ctrl->opts->nr_poll_queues + 1;
55 }
56
nvme_auth_submit(struct nvme_ctrl * ctrl,int qid,void * data,size_t data_len,bool auth_send)57 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
58 void *data, size_t data_len, bool auth_send)
59 {
60 struct nvme_command cmd = {};
61 nvme_submit_flags_t flags = NVME_SUBMIT_RETRY;
62 struct request_queue *q = ctrl->fabrics_q;
63 int ret;
64
65 if (qid != 0) {
66 flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED;
67 q = ctrl->connect_q;
68 }
69
70 cmd.auth_common.opcode = nvme_fabrics_command;
71 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER;
72 cmd.auth_common.spsp0 = 0x01;
73 cmd.auth_common.spsp1 = 0x01;
74 if (auth_send) {
75 cmd.auth_send.fctype = nvme_fabrics_type_auth_send;
76 cmd.auth_send.tl = cpu_to_le32(data_len);
77 } else {
78 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive;
79 cmd.auth_receive.al = cpu_to_le32(data_len);
80 }
81
82 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len,
83 qid == 0 ? NVME_QID_ANY : qid, flags);
84 if (ret > 0)
85 dev_warn(ctrl->device,
86 "qid %d auth_send failed with status %d\n", qid, ret);
87 else if (ret < 0)
88 dev_err(ctrl->device,
89 "qid %d auth_send failed with error %d\n", qid, ret);
90 return ret;
91 }
92
nvme_auth_receive_validate(struct nvme_ctrl * ctrl,int qid,struct nvmf_auth_dhchap_failure_data * data,u16 transaction,u8 expected_msg)93 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,
94 struct nvmf_auth_dhchap_failure_data *data,
95 u16 transaction, u8 expected_msg)
96 {
97 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",
98 __func__, qid, data->auth_type, data->auth_id);
99
100 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES &&
101 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
102 return data->rescode_exp;
103 }
104 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES ||
105 data->auth_id != expected_msg) {
106 dev_warn(ctrl->device,
107 "qid %d invalid message %02x/%02x\n",
108 qid, data->auth_type, data->auth_id);
109 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
110 }
111 if (le16_to_cpu(data->t_id) != transaction) {
112 dev_warn(ctrl->device,
113 "qid %d invalid transaction ID %d\n",
114 qid, le16_to_cpu(data->t_id));
115 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
116 }
117 return 0;
118 }
119
nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)120 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,
121 struct nvme_dhchap_queue_context *chap)
122 {
123 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf;
124 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol);
125
126 if (size > CHAP_BUF_SIZE) {
127 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
128 return -EINVAL;
129 }
130 memset((u8 *)chap->buf, 0, size);
131 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
132 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
133 data->t_id = cpu_to_le16(chap->transaction);
134 data->sc_c = 0; /* No secure channel concatenation */
135 data->napd = 1;
136 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID;
137 data->auth_protocol[0].dhchap.halen = 3;
138 data->auth_protocol[0].dhchap.dhlen = 6;
139 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256;
140 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384;
141 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512;
142 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL;
143 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048;
144 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072;
145 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096;
146 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144;
147 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192;
148
149 return size;
150 }
151
nvme_auth_process_dhchap_challenge(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)152 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,
153 struct nvme_dhchap_queue_context *chap)
154 {
155 struct nvmf_auth_dhchap_challenge_data *data = chap->buf;
156 u16 dhvlen = le16_to_cpu(data->dhvlen);
157 size_t size = sizeof(*data) + data->hl + dhvlen;
158 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid);
159 const char *hmac_name, *kpp_name;
160
161 if (size > CHAP_BUF_SIZE) {
162 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
163 return -EINVAL;
164 }
165
166 hmac_name = nvme_auth_hmac_name(data->hashid);
167 if (!hmac_name) {
168 dev_warn(ctrl->device,
169 "qid %d: invalid HASH ID %d\n",
170 chap->qid, data->hashid);
171 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
172 return -EPROTO;
173 }
174
175 if (chap->hash_id == data->hashid && chap->shash_tfm &&
176 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) &&
177 crypto_shash_digestsize(chap->shash_tfm) == data->hl) {
178 dev_dbg(ctrl->device,
179 "qid %d: reuse existing hash %s\n",
180 chap->qid, hmac_name);
181 goto select_kpp;
182 }
183
184 /* Reset if hash cannot be reused */
185 if (chap->shash_tfm) {
186 crypto_free_shash(chap->shash_tfm);
187 chap->hash_id = 0;
188 chap->hash_len = 0;
189 }
190 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0,
191 CRYPTO_ALG_ALLOCATES_MEMORY);
192 if (IS_ERR(chap->shash_tfm)) {
193 dev_warn(ctrl->device,
194 "qid %d: failed to allocate hash %s, error %ld\n",
195 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm));
196 chap->shash_tfm = NULL;
197 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
198 return -ENOMEM;
199 }
200
201 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) {
202 dev_warn(ctrl->device,
203 "qid %d: invalid hash length %d\n",
204 chap->qid, data->hl);
205 crypto_free_shash(chap->shash_tfm);
206 chap->shash_tfm = NULL;
207 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
208 return -EPROTO;
209 }
210
211 chap->hash_id = data->hashid;
212 chap->hash_len = data->hl;
213 dev_dbg(ctrl->device, "qid %d: selected hash %s\n",
214 chap->qid, hmac_name);
215
216 select_kpp:
217 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid);
218 if (!kpp_name) {
219 dev_warn(ctrl->device,
220 "qid %d: invalid DH group id %d\n",
221 chap->qid, data->dhgid);
222 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
223 /* Leave previous dh_tfm intact */
224 return -EPROTO;
225 }
226
227 if (chap->dhgroup_id == data->dhgid &&
228 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) {
229 dev_dbg(ctrl->device,
230 "qid %d: reuse existing DH group %s\n",
231 chap->qid, gid_name);
232 goto skip_kpp;
233 }
234
235 /* Reset dh_tfm if it can't be reused */
236 if (chap->dh_tfm) {
237 crypto_free_kpp(chap->dh_tfm);
238 chap->dh_tfm = NULL;
239 }
240
241 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) {
242 if (dhvlen == 0) {
243 dev_warn(ctrl->device,
244 "qid %d: empty DH value\n",
245 chap->qid);
246 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
247 return -EPROTO;
248 }
249
250 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0);
251 if (IS_ERR(chap->dh_tfm)) {
252 int ret = PTR_ERR(chap->dh_tfm);
253
254 dev_warn(ctrl->device,
255 "qid %d: error %d initializing DH group %s\n",
256 chap->qid, ret, gid_name);
257 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
258 chap->dh_tfm = NULL;
259 return ret;
260 }
261 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",
262 chap->qid, gid_name);
263 } else if (dhvlen != 0) {
264 dev_warn(ctrl->device,
265 "qid %d: invalid DH value for NULL DH\n",
266 chap->qid);
267 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
268 return -EPROTO;
269 }
270 chap->dhgroup_id = data->dhgid;
271
272 skip_kpp:
273 chap->s1 = le32_to_cpu(data->seqnum);
274 memcpy(chap->c1, data->cval, chap->hash_len);
275 if (dhvlen) {
276 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL);
277 if (!chap->ctrl_key) {
278 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
279 return -ENOMEM;
280 }
281 chap->ctrl_key_len = dhvlen;
282 memcpy(chap->ctrl_key, data->cval + chap->hash_len,
283 dhvlen);
284 dev_dbg(ctrl->device, "ctrl public key %*ph\n",
285 (int)chap->ctrl_key_len, chap->ctrl_key);
286 }
287
288 return 0;
289 }
290
nvme_auth_set_dhchap_reply_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)291 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,
292 struct nvme_dhchap_queue_context *chap)
293 {
294 struct nvmf_auth_dhchap_reply_data *data = chap->buf;
295 size_t size = sizeof(*data);
296
297 size += 2 * chap->hash_len;
298
299 if (chap->host_key_len)
300 size += chap->host_key_len;
301
302 if (size > CHAP_BUF_SIZE) {
303 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
304 return -EINVAL;
305 }
306
307 memset(chap->buf, 0, size);
308 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
309 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
310 data->t_id = cpu_to_le16(chap->transaction);
311 data->hl = chap->hash_len;
312 data->dhvlen = cpu_to_le16(chap->host_key_len);
313 memcpy(data->rval, chap->response, chap->hash_len);
314 if (ctrl->ctrl_key) {
315 chap->bi_directional = true;
316 get_random_bytes(chap->c2, chap->hash_len);
317 data->cvalid = 1;
318 memcpy(data->rval + chap->hash_len, chap->c2,
319 chap->hash_len);
320 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",
321 __func__, chap->qid, (int)chap->hash_len, chap->c2);
322 } else {
323 memset(chap->c2, 0, chap->hash_len);
324 }
325 chap->s2 = nvme_auth_get_seqnum();
326 data->seqnum = cpu_to_le32(chap->s2);
327 if (chap->host_key_len) {
328 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",
329 __func__, chap->qid,
330 chap->host_key_len, chap->host_key);
331 memcpy(data->rval + 2 * chap->hash_len, chap->host_key,
332 chap->host_key_len);
333 }
334
335 return size;
336 }
337
nvme_auth_process_dhchap_success1(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)338 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,
339 struct nvme_dhchap_queue_context *chap)
340 {
341 struct nvmf_auth_dhchap_success1_data *data = chap->buf;
342 size_t size = sizeof(*data) + chap->hash_len;
343
344 if (size > CHAP_BUF_SIZE) {
345 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
346 return -EINVAL;
347 }
348
349 if (data->hl != chap->hash_len) {
350 dev_warn(ctrl->device,
351 "qid %d: invalid hash length %u\n",
352 chap->qid, data->hl);
353 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
354 return -EPROTO;
355 }
356
357 /* Just print out information for the admin queue */
358 if (chap->qid == 0)
359 dev_info(ctrl->device,
360 "qid 0: authenticated with hash %s dhgroup %s\n",
361 nvme_auth_hmac_name(chap->hash_id),
362 nvme_auth_dhgroup_name(chap->dhgroup_id));
363
364 if (!data->rvalid)
365 return 0;
366
367 /* Validate controller response */
368 if (memcmp(chap->response, data->rval, data->hl)) {
369 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",
370 __func__, chap->qid, (int)chap->hash_len, data->rval);
371 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",
372 __func__, chap->qid, (int)chap->hash_len,
373 chap->response);
374 dev_warn(ctrl->device,
375 "qid %d: controller authentication failed\n",
376 chap->qid);
377 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
378 return -ECONNREFUSED;
379 }
380
381 /* Just print out information for the admin queue */
382 if (chap->qid == 0)
383 dev_info(ctrl->device,
384 "qid 0: controller authenticated\n");
385 return 0;
386 }
387
nvme_auth_set_dhchap_success2_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)388 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,
389 struct nvme_dhchap_queue_context *chap)
390 {
391 struct nvmf_auth_dhchap_success2_data *data = chap->buf;
392 size_t size = sizeof(*data);
393
394 memset(chap->buf, 0, size);
395 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
396 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
397 data->t_id = cpu_to_le16(chap->transaction);
398
399 return size;
400 }
401
nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)402 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,
403 struct nvme_dhchap_queue_context *chap)
404 {
405 struct nvmf_auth_dhchap_failure_data *data = chap->buf;
406 size_t size = sizeof(*data);
407
408 memset(chap->buf, 0, size);
409 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
410 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
411 data->t_id = cpu_to_le16(chap->transaction);
412 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
413 data->rescode_exp = chap->status;
414
415 return size;
416 }
417
nvme_auth_dhchap_setup_host_response(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)418 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,
419 struct nvme_dhchap_queue_context *chap)
420 {
421 SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
422 u8 buf[4], *challenge = chap->c1;
423 int ret;
424
425 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",
426 __func__, chap->qid, chap->s1, chap->transaction);
427
428 if (!chap->transformed_key) {
429 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key,
430 ctrl->opts->host->nqn);
431 if (IS_ERR(chap->transformed_key)) {
432 ret = PTR_ERR(chap->transformed_key);
433 chap->transformed_key = NULL;
434 return ret;
435 }
436 } else {
437 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",
438 __func__, chap->qid);
439 }
440
441 ret = crypto_shash_setkey(chap->shash_tfm,
442 chap->transformed_key->key, chap->transformed_key->len);
443 if (ret) {
444 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
445 chap->qid, ret);
446 goto out;
447 }
448
449 if (chap->dh_tfm) {
450 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
451 if (!challenge) {
452 ret = -ENOMEM;
453 goto out;
454 }
455 ret = nvme_auth_augmented_challenge(chap->hash_id,
456 chap->sess_key,
457 chap->sess_key_len,
458 chap->c1, challenge,
459 chap->hash_len);
460 if (ret)
461 goto out;
462 }
463
464 shash->tfm = chap->shash_tfm;
465 ret = crypto_shash_init(shash);
466 if (ret)
467 goto out;
468 ret = crypto_shash_update(shash, challenge, chap->hash_len);
469 if (ret)
470 goto out;
471 put_unaligned_le32(chap->s1, buf);
472 ret = crypto_shash_update(shash, buf, 4);
473 if (ret)
474 goto out;
475 put_unaligned_le16(chap->transaction, buf);
476 ret = crypto_shash_update(shash, buf, 2);
477 if (ret)
478 goto out;
479 memset(buf, 0, sizeof(buf));
480 ret = crypto_shash_update(shash, buf, 1);
481 if (ret)
482 goto out;
483 ret = crypto_shash_update(shash, "HostHost", 8);
484 if (ret)
485 goto out;
486 ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
487 strlen(ctrl->opts->host->nqn));
488 if (ret)
489 goto out;
490 ret = crypto_shash_update(shash, buf, 1);
491 if (ret)
492 goto out;
493 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
494 strlen(ctrl->opts->subsysnqn));
495 if (ret)
496 goto out;
497 ret = crypto_shash_final(shash, chap->response);
498 out:
499 if (challenge != chap->c1)
500 kfree(challenge);
501 return ret;
502 }
503
nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)504 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,
505 struct nvme_dhchap_queue_context *chap)
506 {
507 SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
508 struct nvme_dhchap_key *transformed_key;
509 u8 buf[4], *challenge = chap->c2;
510 int ret;
511
512 transformed_key = nvme_auth_transform_key(ctrl->ctrl_key,
513 ctrl->opts->subsysnqn);
514 if (IS_ERR(transformed_key)) {
515 ret = PTR_ERR(transformed_key);
516 return ret;
517 }
518
519 ret = crypto_shash_setkey(chap->shash_tfm,
520 transformed_key->key, transformed_key->len);
521 if (ret) {
522 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
523 chap->qid, ret);
524 goto out;
525 }
526
527 if (chap->dh_tfm) {
528 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
529 if (!challenge) {
530 ret = -ENOMEM;
531 goto out;
532 }
533 ret = nvme_auth_augmented_challenge(chap->hash_id,
534 chap->sess_key,
535 chap->sess_key_len,
536 chap->c2, challenge,
537 chap->hash_len);
538 if (ret)
539 goto out;
540 }
541 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",
542 __func__, chap->qid, chap->s2, chap->transaction);
543 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",
544 __func__, chap->qid, (int)chap->hash_len, challenge);
545 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",
546 __func__, chap->qid, ctrl->opts->subsysnqn);
547 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",
548 __func__, chap->qid, ctrl->opts->host->nqn);
549 shash->tfm = chap->shash_tfm;
550 ret = crypto_shash_init(shash);
551 if (ret)
552 goto out;
553 ret = crypto_shash_update(shash, challenge, chap->hash_len);
554 if (ret)
555 goto out;
556 put_unaligned_le32(chap->s2, buf);
557 ret = crypto_shash_update(shash, buf, 4);
558 if (ret)
559 goto out;
560 put_unaligned_le16(chap->transaction, buf);
561 ret = crypto_shash_update(shash, buf, 2);
562 if (ret)
563 goto out;
564 memset(buf, 0, 4);
565 ret = crypto_shash_update(shash, buf, 1);
566 if (ret)
567 goto out;
568 ret = crypto_shash_update(shash, "Controller", 10);
569 if (ret)
570 goto out;
571 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
572 strlen(ctrl->opts->subsysnqn));
573 if (ret)
574 goto out;
575 ret = crypto_shash_update(shash, buf, 1);
576 if (ret)
577 goto out;
578 ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
579 strlen(ctrl->opts->host->nqn));
580 if (ret)
581 goto out;
582 ret = crypto_shash_final(shash, chap->response);
583 out:
584 if (challenge != chap->c2)
585 kfree(challenge);
586 nvme_auth_free_key(transformed_key);
587 return ret;
588 }
589
nvme_auth_dhchap_exponential(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)590 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
591 struct nvme_dhchap_queue_context *chap)
592 {
593 int ret;
594
595 if (chap->host_key && chap->host_key_len) {
596 dev_dbg(ctrl->device,
597 "qid %d: reusing host key\n", chap->qid);
598 goto gen_sesskey;
599 }
600 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id);
601 if (ret < 0) {
602 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
603 return ret;
604 }
605
606 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm);
607
608 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL);
609 if (!chap->host_key) {
610 chap->host_key_len = 0;
611 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
612 return -ENOMEM;
613 }
614 ret = nvme_auth_gen_pubkey(chap->dh_tfm,
615 chap->host_key, chap->host_key_len);
616 if (ret) {
617 dev_dbg(ctrl->device,
618 "failed to generate public key, error %d\n", ret);
619 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
620 return ret;
621 }
622
623 gen_sesskey:
624 chap->sess_key_len = chap->host_key_len;
625 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL);
626 if (!chap->sess_key) {
627 chap->sess_key_len = 0;
628 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
629 return -ENOMEM;
630 }
631
632 ret = nvme_auth_gen_shared_secret(chap->dh_tfm,
633 chap->ctrl_key, chap->ctrl_key_len,
634 chap->sess_key, chap->sess_key_len);
635 if (ret) {
636 dev_dbg(ctrl->device,
637 "failed to generate shared secret, error %d\n", ret);
638 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
639 return ret;
640 }
641 dev_dbg(ctrl->device, "shared secret %*ph\n",
642 (int)chap->sess_key_len, chap->sess_key);
643 return 0;
644 }
645
nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context * chap)646 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap)
647 {
648 nvme_auth_free_key(chap->transformed_key);
649 chap->transformed_key = NULL;
650 kfree_sensitive(chap->host_key);
651 chap->host_key = NULL;
652 chap->host_key_len = 0;
653 kfree_sensitive(chap->ctrl_key);
654 chap->ctrl_key = NULL;
655 chap->ctrl_key_len = 0;
656 kfree_sensitive(chap->sess_key);
657 chap->sess_key = NULL;
658 chap->sess_key_len = 0;
659 chap->status = 0;
660 chap->error = 0;
661 chap->s1 = 0;
662 chap->s2 = 0;
663 chap->bi_directional = false;
664 chap->transaction = 0;
665 memset(chap->c1, 0, sizeof(chap->c1));
666 memset(chap->c2, 0, sizeof(chap->c2));
667 mempool_free(chap->buf, nvme_chap_buf_pool);
668 chap->buf = NULL;
669 }
670
nvme_auth_free_dhchap(struct nvme_dhchap_queue_context * chap)671 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap)
672 {
673 nvme_auth_reset_dhchap(chap);
674 if (chap->shash_tfm)
675 crypto_free_shash(chap->shash_tfm);
676 if (chap->dh_tfm)
677 crypto_free_kpp(chap->dh_tfm);
678 }
679
nvme_queue_auth_work(struct work_struct * work)680 static void nvme_queue_auth_work(struct work_struct *work)
681 {
682 struct nvme_dhchap_queue_context *chap =
683 container_of(work, struct nvme_dhchap_queue_context, auth_work);
684 struct nvme_ctrl *ctrl = chap->ctrl;
685 size_t tl;
686 int ret = 0;
687
688 /*
689 * Allocate a large enough buffer for the entire negotiation:
690 * 4k is enough to ffdhe8192.
691 */
692 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL);
693 if (!chap->buf) {
694 chap->error = -ENOMEM;
695 return;
696 }
697
698 chap->transaction = ctrl->transaction++;
699
700 /* DH-HMAC-CHAP Step 1: send negotiate */
701 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",
702 __func__, chap->qid);
703 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);
704 if (ret < 0) {
705 chap->error = ret;
706 return;
707 }
708 tl = ret;
709 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
710 if (ret) {
711 chap->error = ret;
712 return;
713 }
714
715 /* DH-HMAC-CHAP Step 2: receive challenge */
716 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",
717 __func__, chap->qid);
718
719 memset(chap->buf, 0, CHAP_BUF_SIZE);
720 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
721 false);
722 if (ret) {
723 dev_warn(ctrl->device,
724 "qid %d failed to receive challenge, %s %d\n",
725 chap->qid, ret < 0 ? "error" : "nvme status", ret);
726 chap->error = ret;
727 return;
728 }
729 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,
730 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE);
731 if (ret) {
732 chap->status = ret;
733 chap->error = -EKEYREJECTED;
734 return;
735 }
736
737 ret = nvme_auth_process_dhchap_challenge(ctrl, chap);
738 if (ret) {
739 /* Invalid challenge parameters */
740 chap->error = ret;
741 goto fail2;
742 }
743
744 if (chap->ctrl_key_len) {
745 dev_dbg(ctrl->device,
746 "%s: qid %d DH exponential\n",
747 __func__, chap->qid);
748 ret = nvme_auth_dhchap_exponential(ctrl, chap);
749 if (ret) {
750 chap->error = ret;
751 goto fail2;
752 }
753 }
754
755 dev_dbg(ctrl->device, "%s: qid %d host response\n",
756 __func__, chap->qid);
757 mutex_lock(&ctrl->dhchap_auth_mutex);
758 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);
759 mutex_unlock(&ctrl->dhchap_auth_mutex);
760 if (ret) {
761 chap->error = ret;
762 goto fail2;
763 }
764
765 /* DH-HMAC-CHAP Step 3: send reply */
766 dev_dbg(ctrl->device, "%s: qid %d send reply\n",
767 __func__, chap->qid);
768 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);
769 if (ret < 0) {
770 chap->error = ret;
771 goto fail2;
772 }
773
774 tl = ret;
775 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
776 if (ret) {
777 chap->error = ret;
778 goto fail2;
779 }
780
781 /* DH-HMAC-CHAP Step 4: receive success1 */
782 dev_dbg(ctrl->device, "%s: qid %d receive success1\n",
783 __func__, chap->qid);
784
785 memset(chap->buf, 0, CHAP_BUF_SIZE);
786 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
787 false);
788 if (ret) {
789 dev_warn(ctrl->device,
790 "qid %d failed to receive success1, %s %d\n",
791 chap->qid, ret < 0 ? "error" : "nvme status", ret);
792 chap->error = ret;
793 return;
794 }
795 ret = nvme_auth_receive_validate(ctrl, chap->qid,
796 chap->buf, chap->transaction,
797 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1);
798 if (ret) {
799 chap->status = ret;
800 chap->error = -EKEYREJECTED;
801 return;
802 }
803
804 mutex_lock(&ctrl->dhchap_auth_mutex);
805 if (ctrl->ctrl_key) {
806 dev_dbg(ctrl->device,
807 "%s: qid %d controller response\n",
808 __func__, chap->qid);
809 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);
810 if (ret) {
811 mutex_unlock(&ctrl->dhchap_auth_mutex);
812 chap->error = ret;
813 goto fail2;
814 }
815 }
816 mutex_unlock(&ctrl->dhchap_auth_mutex);
817
818 ret = nvme_auth_process_dhchap_success1(ctrl, chap);
819 if (ret) {
820 /* Controller authentication failed */
821 chap->error = -EKEYREJECTED;
822 goto fail2;
823 }
824
825 if (chap->bi_directional) {
826 /* DH-HMAC-CHAP Step 5: send success2 */
827 dev_dbg(ctrl->device, "%s: qid %d send success2\n",
828 __func__, chap->qid);
829 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);
830 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
831 if (ret)
832 chap->error = ret;
833 }
834 if (!ret) {
835 chap->error = 0;
836 return;
837 }
838
839 fail2:
840 if (chap->status == 0)
841 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
842 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",
843 __func__, chap->qid, chap->status);
844 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);
845 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
846 /*
847 * only update error if send failure2 failed and no other
848 * error had been set during authentication.
849 */
850 if (ret && !chap->error)
851 chap->error = ret;
852 }
853
nvme_auth_negotiate(struct nvme_ctrl * ctrl,int qid)854 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)
855 {
856 struct nvme_dhchap_queue_context *chap;
857
858 if (!ctrl->host_key) {
859 dev_warn(ctrl->device, "qid %d: no key\n", qid);
860 return -ENOKEY;
861 }
862
863 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {
864 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);
865 return -ENOKEY;
866 }
867
868 chap = &ctrl->dhchap_ctxs[qid];
869 cancel_work_sync(&chap->auth_work);
870 queue_work(nvme_auth_wq, &chap->auth_work);
871 return 0;
872 }
873 EXPORT_SYMBOL_GPL(nvme_auth_negotiate);
874
nvme_auth_wait(struct nvme_ctrl * ctrl,int qid)875 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)
876 {
877 struct nvme_dhchap_queue_context *chap;
878 int ret;
879
880 chap = &ctrl->dhchap_ctxs[qid];
881 flush_work(&chap->auth_work);
882 ret = chap->error;
883 /* clear sensitive info */
884 nvme_auth_reset_dhchap(chap);
885 return ret;
886 }
887 EXPORT_SYMBOL_GPL(nvme_auth_wait);
888
nvme_ctrl_auth_work(struct work_struct * work)889 static void nvme_ctrl_auth_work(struct work_struct *work)
890 {
891 struct nvme_ctrl *ctrl =
892 container_of(work, struct nvme_ctrl, dhchap_auth_work);
893 int ret, q;
894
895 /*
896 * If the ctrl is no connected, bail as reconnect will handle
897 * authentication.
898 */
899 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
900 return;
901
902 /* Authenticate admin queue first */
903 ret = nvme_auth_negotiate(ctrl, 0);
904 if (ret) {
905 dev_warn(ctrl->device,
906 "qid 0: error %d setting up authentication\n", ret);
907 return;
908 }
909 ret = nvme_auth_wait(ctrl, 0);
910 if (ret) {
911 dev_warn(ctrl->device,
912 "qid 0: authentication failed\n");
913 return;
914 }
915
916 for (q = 1; q < ctrl->queue_count; q++) {
917 ret = nvme_auth_negotiate(ctrl, q);
918 if (ret) {
919 dev_warn(ctrl->device,
920 "qid %d: error %d setting up authentication\n",
921 q, ret);
922 break;
923 }
924 }
925
926 /*
927 * Failure is a soft-state; credentials remain valid until
928 * the controller terminates the connection.
929 */
930 for (q = 1; q < ctrl->queue_count; q++) {
931 ret = nvme_auth_wait(ctrl, q);
932 if (ret)
933 dev_warn(ctrl->device,
934 "qid %d: authentication failed\n", q);
935 }
936 }
937
nvme_auth_init_ctrl(struct nvme_ctrl * ctrl)938 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)
939 {
940 struct nvme_dhchap_queue_context *chap;
941 int i, ret;
942
943 mutex_init(&ctrl->dhchap_auth_mutex);
944 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);
945 if (!ctrl->opts)
946 return 0;
947 ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret,
948 &ctrl->host_key);
949 if (ret)
950 return ret;
951 ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret,
952 &ctrl->ctrl_key);
953 if (ret)
954 goto err_free_dhchap_secret;
955
956 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)
957 return 0;
958
959 ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl),
960 sizeof(*chap), GFP_KERNEL);
961 if (!ctrl->dhchap_ctxs) {
962 ret = -ENOMEM;
963 goto err_free_dhchap_ctrl_secret;
964 }
965
966 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {
967 chap = &ctrl->dhchap_ctxs[i];
968 chap->qid = i;
969 chap->ctrl = ctrl;
970 INIT_WORK(&chap->auth_work, nvme_queue_auth_work);
971 }
972
973 return 0;
974 err_free_dhchap_ctrl_secret:
975 nvme_auth_free_key(ctrl->ctrl_key);
976 ctrl->ctrl_key = NULL;
977 err_free_dhchap_secret:
978 nvme_auth_free_key(ctrl->host_key);
979 ctrl->host_key = NULL;
980 return ret;
981 }
982 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl);
983
nvme_auth_stop(struct nvme_ctrl * ctrl)984 void nvme_auth_stop(struct nvme_ctrl *ctrl)
985 {
986 cancel_work_sync(&ctrl->dhchap_auth_work);
987 }
988 EXPORT_SYMBOL_GPL(nvme_auth_stop);
989
nvme_auth_free(struct nvme_ctrl * ctrl)990 void nvme_auth_free(struct nvme_ctrl *ctrl)
991 {
992 int i;
993
994 if (ctrl->dhchap_ctxs) {
995 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)
996 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);
997 kfree(ctrl->dhchap_ctxs);
998 }
999 if (ctrl->host_key) {
1000 nvme_auth_free_key(ctrl->host_key);
1001 ctrl->host_key = NULL;
1002 }
1003 if (ctrl->ctrl_key) {
1004 nvme_auth_free_key(ctrl->ctrl_key);
1005 ctrl->ctrl_key = NULL;
1006 }
1007 }
1008 EXPORT_SYMBOL_GPL(nvme_auth_free);
1009
nvme_init_auth(void)1010 int __init nvme_init_auth(void)
1011 {
1012 nvme_auth_wq = alloc_workqueue("nvme-auth-wq",
1013 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
1014 if (!nvme_auth_wq)
1015 return -ENOMEM;
1016
1017 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache",
1018 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL);
1019 if (!nvme_chap_buf_cache)
1020 goto err_destroy_workqueue;
1021
1022 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab,
1023 mempool_free_slab, nvme_chap_buf_cache);
1024 if (!nvme_chap_buf_pool)
1025 goto err_destroy_chap_buf_cache;
1026
1027 return 0;
1028 err_destroy_chap_buf_cache:
1029 kmem_cache_destroy(nvme_chap_buf_cache);
1030 err_destroy_workqueue:
1031 destroy_workqueue(nvme_auth_wq);
1032 return -ENOMEM;
1033 }
1034
nvme_exit_auth(void)1035 void __exit nvme_exit_auth(void)
1036 {
1037 mempool_destroy(nvme_chap_buf_pool);
1038 kmem_cache_destroy(nvme_chap_buf_cache);
1039 destroy_workqueue(nvme_auth_wq);
1040 }
1041