60 	int (*complete)(struct aead_request *req, u32 flags);
82 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
85 	struct aead_request *req)  in crypto_gcm_reqctx()  argument
87 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_gcm_reqctx()
89 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_gcm_reqctx()
105 		struct skcipher_request req;  in crypto_gcm_setkey()  member
123 	skcipher_request_set_tfm(&data->req, ctr);  in crypto_gcm_setkey()
124 	skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |  in crypto_gcm_setkey()
128 	skcipher_request_set_crypt(&data->req, data->sg, data->sg,  in crypto_gcm_setkey()
131 	err = crypto_wait_req(crypto_skcipher_encrypt(&data->req),  in crypto_gcm_setkey()
152 static void crypto_gcm_init_common(struct aead_request *req)  in crypto_gcm_init_common()  argument
154 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_common()
159 	memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE);  in crypto_gcm_init_common()
164 	sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);  in crypto_gcm_init_common()
168 	if (req->src != req->dst) {  in crypto_gcm_init_common()
171 		sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);  in crypto_gcm_init_common()
177 static void crypto_gcm_init_crypt(struct aead_request *req,  in crypto_gcm_init_crypt()  argument
180 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_init_crypt()
182 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_crypt()
186 	dst = req->src == req->dst ? pctx->src : pctx->dst;  in crypto_gcm_init_crypt()
202 static int gcm_hash_update(struct aead_request *req,  in gcm_hash_update()  argument
207 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_update()
210 	ahash_request_set_callback(ahreq, flags, compl, req);  in gcm_hash_update()
216 static int gcm_hash_remain(struct aead_request *req,  in gcm_hash_remain()  argument
220 	return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);  in gcm_hash_remain()
223 static int gcm_hash_len(struct aead_request *req, u32 flags)  in gcm_hash_len()  argument
225 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len()
230 	lengths.a = cpu_to_be64(req->assoclen * 8);  in gcm_hash_len()
234 	ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);  in gcm_hash_len()
241 static int gcm_hash_len_continue(struct aead_request *req, u32 flags)  in gcm_hash_len_continue()  argument
243 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len_continue()
246 	return gctx->complete(req, flags);  in gcm_hash_len_continue()
251 	struct aead_request *req = data;  in gcm_hash_len_done()  local
256 	err = gcm_hash_len_continue(req, 0);  in gcm_hash_len_done()
261 	aead_request_complete(req, err);  in gcm_hash_len_done()
264 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_remain_continue()  argument
266 	return gcm_hash_len(req, flags) ?:  in gcm_hash_crypt_remain_continue()
267 	       gcm_hash_len_continue(req, flags);  in gcm_hash_crypt_remain_continue()
272 	struct aead_request *req = data;  in gcm_hash_crypt_remain_done()  local
277 	err = gcm_hash_crypt_remain_continue(req, 0);  in gcm_hash_crypt_remain_done()
282 	aead_request_complete(req, err);  in gcm_hash_crypt_remain_done()
285 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_continue()  argument
287 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_crypt_continue()
293 		return gcm_hash_remain(req, remain,  in gcm_hash_crypt_continue()
295 		       gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
297 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
302 	struct aead_request *req = data;  in gcm_hash_crypt_done()  local
307 	err = gcm_hash_crypt_continue(req, 0);  in gcm_hash_crypt_done()
312 	aead_request_complete(req, err);  in gcm_hash_crypt_done()
315 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_remain_continue()  argument
317 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_assoc_remain_continue()
321 		return gcm_hash_update(req, gcm_hash_crypt_done,  in gcm_hash_assoc_remain_continue()
323 		       gcm_hash_crypt_continue(req, flags);  in gcm_hash_assoc_remain_continue()
325 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_assoc_remain_continue()
330 	struct aead_request *req = data;  in gcm_hash_assoc_remain_done()  local
335 	err = gcm_hash_assoc_remain_continue(req, 0);  in gcm_hash_assoc_remain_done()
340 	aead_request_complete(req, err);  in gcm_hash_assoc_remain_done()
343 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_continue()  argument
347 	remain = gcm_remain(req->assoclen);  in gcm_hash_assoc_continue()
349 		return gcm_hash_remain(req, remain,  in gcm_hash_assoc_continue()
351 		       gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
353 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
358 	struct aead_request *req = data;  in gcm_hash_assoc_done()  local
363 	err = gcm_hash_assoc_continue(req, 0);  in gcm_hash_assoc_done()
368 	aead_request_complete(req, err);  in gcm_hash_assoc_done()
371 static int gcm_hash_init_continue(struct aead_request *req, u32 flags)  in gcm_hash_init_continue()  argument
373 	if (req->assoclen)  in gcm_hash_init_continue()
374 		return gcm_hash_update(req, gcm_hash_assoc_done,  in gcm_hash_init_continue()
375 				       req->src, req->assoclen, flags) ?:  in gcm_hash_init_continue()
376 		       gcm_hash_assoc_continue(req, flags);  in gcm_hash_init_continue()
378 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_init_continue()
383 	struct aead_request *req = data;  in gcm_hash_init_done()  local
388 	err = gcm_hash_init_continue(req, 0);  in gcm_hash_init_done()
393 	aead_request_complete(req, err);  in gcm_hash_init_done()
396 static int gcm_hash(struct aead_request *req, u32 flags)  in gcm_hash()  argument
398 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash()
400 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in gcm_hash()
404 	ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);  in gcm_hash()
406 	       gcm_hash_init_continue(req, flags);  in gcm_hash()
409 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)  in gcm_enc_copy_hash()  argument
411 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_enc_copy_hash()
412 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in gcm_enc_copy_hash()
416 	scatterwalk_map_and_copy(auth_tag, req->dst,  in gcm_enc_copy_hash()
417 				 req->assoclen + req->cryptlen,  in gcm_enc_copy_hash()
422 static int gcm_encrypt_continue(struct aead_request *req, u32 flags)  in gcm_encrypt_continue()  argument
424 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_encrypt_continue()
427 	gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);  in gcm_encrypt_continue()
428 	gctx->cryptlen = req->cryptlen;  in gcm_encrypt_continue()
431 	return gcm_hash(req, flags);  in gcm_encrypt_continue()
436 	struct aead_request *req = data;  in gcm_encrypt_done()  local
441 	err = gcm_encrypt_continue(req, 0);  in gcm_encrypt_done()
446 	aead_request_complete(req, err);  in gcm_encrypt_done()
449 static int crypto_gcm_encrypt(struct aead_request *req)  in crypto_gcm_encrypt()  argument
451 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_encrypt()
453 	u32 flags = aead_request_flags(req);  in crypto_gcm_encrypt()
455 	crypto_gcm_init_common(req);  in crypto_gcm_encrypt()
456 	crypto_gcm_init_crypt(req, req->cryptlen);  in crypto_gcm_encrypt()
457 	skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);  in crypto_gcm_encrypt()
460 	       gcm_encrypt_continue(req, flags);  in crypto_gcm_encrypt()
463 static int crypto_gcm_verify(struct aead_request *req)  in crypto_gcm_verify()  argument
465 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_verify()
466 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_verify()
470 	unsigned int cryptlen = req->cryptlen - authsize;  in crypto_gcm_verify()
473 	scatterwalk_map_and_copy(iauth_tag, req->src,  in crypto_gcm_verify()
474 				 req->assoclen + cryptlen, authsize, 0);  in crypto_gcm_verify()
480 	struct aead_request *req = data;  in gcm_decrypt_done()  local
483 		err = crypto_gcm_verify(req);  in gcm_decrypt_done()
485 	aead_request_complete(req, err);  in gcm_decrypt_done()
488 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)  in gcm_dec_hash_continue()  argument
490 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_dec_hash_continue()
494 	crypto_gcm_init_crypt(req, gctx->cryptlen);  in gcm_dec_hash_continue()
495 	skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);  in gcm_dec_hash_continue()
496 	return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);  in gcm_dec_hash_continue()
499 static int crypto_gcm_decrypt(struct aead_request *req)  in crypto_gcm_decrypt()  argument
501 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_decrypt()
502 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_decrypt()
505 	unsigned int cryptlen = req->cryptlen;  in crypto_gcm_decrypt()
506 	u32 flags = aead_request_flags(req);  in crypto_gcm_decrypt()
510 	crypto_gcm_init_common(req);  in crypto_gcm_decrypt()
516 	return gcm_hash(req, flags);  in crypto_gcm_decrypt()
718 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)  in crypto_rfc4106_crypt()  argument
720 	struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4106_crypt()
721 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4106_crypt()
729 	scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0);  in crypto_rfc4106_crypt()
732 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4106_crypt()
735 	sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8);  in crypto_rfc4106_crypt()
736 	sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);  in crypto_rfc4106_crypt()
740 	if (req->src != req->dst) {  in crypto_rfc4106_crypt()
742 		sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8);  in crypto_rfc4106_crypt()
743 		sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);  in crypto_rfc4106_crypt()
749 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,  in crypto_rfc4106_crypt()
750 				  req->base.data);  in crypto_rfc4106_crypt()
752 			       req->src == req->dst ? rctx->src : rctx->dst,  in crypto_rfc4106_crypt()
753 			       req->cryptlen, iv);  in crypto_rfc4106_crypt()
754 	aead_request_set_ad(subreq, req->assoclen - 8);  in crypto_rfc4106_crypt()
759 static int crypto_rfc4106_encrypt(struct aead_request *req)  in crypto_rfc4106_encrypt()  argument
763 	err = crypto_ipsec_check_assoclen(req->assoclen);  in crypto_rfc4106_encrypt()
767 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_encrypt()
769 	return crypto_aead_encrypt(req);  in crypto_rfc4106_encrypt()
772 static int crypto_rfc4106_decrypt(struct aead_request *req)  in crypto_rfc4106_decrypt()  argument
776 	err = crypto_ipsec_check_assoclen(req->assoclen);  in crypto_rfc4106_decrypt()
780 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_decrypt()
782 	return crypto_aead_decrypt(req);  in crypto_rfc4106_decrypt()
924 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)  in crypto_rfc4543_crypt()  argument
926 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_crypt()
928 	struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4543_crypt()
935 	if (req->src != req->dst) {  in crypto_rfc4543_crypt()
936 		err = crypto_rfc4543_copy_src_to_dst(req, enc);  in crypto_rfc4543_crypt()
942 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4543_crypt()
945 	aead_request_set_callback(subreq, req->base.flags,  in crypto_rfc4543_crypt()
946 				  req->base.complete, req->base.data);  in crypto_rfc4543_crypt()
947 	aead_request_set_crypt(subreq, req->src, req->dst,  in crypto_rfc4543_crypt()
949 	aead_request_set_ad(subreq, req->assoclen + req->cryptlen -  in crypto_rfc4543_crypt()
955 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)  in crypto_rfc4543_copy_src_to_dst()  argument
957 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_copy_src_to_dst()
960 	unsigned int nbytes = req->assoclen + req->cryptlen -  in crypto_rfc4543_copy_src_to_dst()
965 	skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);  in crypto_rfc4543_copy_src_to_dst()
966 	skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);  in crypto_rfc4543_copy_src_to_dst()
971 static int crypto_rfc4543_encrypt(struct aead_request *req)  in crypto_rfc4543_encrypt()  argument
973 	return crypto_ipsec_check_assoclen(req->assoclen) ?:  in crypto_rfc4543_encrypt()
974 	       crypto_rfc4543_crypt(req, true);  in crypto_rfc4543_encrypt()
977 static int crypto_rfc4543_decrypt(struct aead_request *req)  in crypto_rfc4543_decrypt()  argument
979 	return crypto_ipsec_check_assoclen(req->assoclen) ?:  in crypto_rfc4543_decrypt()
980 	       crypto_rfc4543_crypt(req, false);  in crypto_rfc4543_decrypt()