109 static int ceph_tcp_recv(struct ceph_connection *con)  in ceph_tcp_recv()  argument
113 	dout("%s con %p %s %zu\n", __func__, con,  in ceph_tcp_recv()
114 	     iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",  in ceph_tcp_recv()
115 	     iov_iter_count(&con->v2.in_iter));  in ceph_tcp_recv()
116 	ret = do_recvmsg(con->sock, &con->v2.in_iter);  in ceph_tcp_recv()
117 	dout("%s con %p ret %d left %zu\n", __func__, con, ret,  in ceph_tcp_recv()
118 	     iov_iter_count(&con->v2.in_iter));  in ceph_tcp_recv()
196 static int ceph_tcp_send(struct ceph_connection *con)  in ceph_tcp_send()  argument
200 	dout("%s con %p have %zu try_sendpage %d\n", __func__, con,  in ceph_tcp_send()
201 	     iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);  in ceph_tcp_send()
202 	if (con->v2.out_iter_sendpage)  in ceph_tcp_send()
203 		ret = do_try_sendpage(con->sock, &con->v2.out_iter);  in ceph_tcp_send()
205 		ret = do_sendmsg(con->sock, &con->v2.out_iter);  in ceph_tcp_send()
206 	dout("%s con %p ret %d left %zu\n", __func__, con, ret,  in ceph_tcp_send()
207 	     iov_iter_count(&con->v2.out_iter));  in ceph_tcp_send()
211 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)  in add_in_kvec()  argument
213 	BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));  in add_in_kvec()
214 	WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));  in add_in_kvec()
216 	con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;  in add_in_kvec()
217 	con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;  in add_in_kvec()
218 	con->v2.in_kvec_cnt++;  in add_in_kvec()
220 	con->v2.in_iter.nr_segs++;  in add_in_kvec()
221 	con->v2.in_iter.count += len;  in add_in_kvec()
224 static void reset_in_kvecs(struct ceph_connection *con)  in reset_in_kvecs()  argument
226 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in reset_in_kvecs()
228 	con->v2.in_kvec_cnt = 0;  in reset_in_kvecs()
229 	iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);  in reset_in_kvecs()
232 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)  in set_in_bvec()  argument
234 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in set_in_bvec()
236 	con->v2.in_bvec = *bv;  in set_in_bvec()
237 	iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);  in set_in_bvec()
240 static void set_in_skip(struct ceph_connection *con, int len)  in set_in_skip()  argument
242 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in set_in_skip()
244 	dout("%s con %p len %d\n", __func__, con, len);  in set_in_skip()
245 	iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);  in set_in_skip()
248 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)  in add_out_kvec()  argument
250 	BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));  in add_out_kvec()
251 	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));  in add_out_kvec()
252 	WARN_ON(con->v2.out_zero);  in add_out_kvec()
254 	con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;  in add_out_kvec()
255 	con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;  in add_out_kvec()
256 	con->v2.out_kvec_cnt++;  in add_out_kvec()
258 	con->v2.out_iter.nr_segs++;  in add_out_kvec()
259 	con->v2.out_iter.count += len;  in add_out_kvec()
262 static void reset_out_kvecs(struct ceph_connection *con)  in reset_out_kvecs()  argument
264 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in reset_out_kvecs()
265 	WARN_ON(con->v2.out_zero);  in reset_out_kvecs()
267 	con->v2.out_kvec_cnt = 0;  in reset_out_kvecs()
269 	iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);  in reset_out_kvecs()
270 	con->v2.out_iter_sendpage = false;  in reset_out_kvecs()
273 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,  in set_out_bvec()  argument
276 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in set_out_bvec()
277 	WARN_ON(con->v2.out_zero);  in set_out_bvec()
279 	con->v2.out_bvec = *bv;  in set_out_bvec()
280 	con->v2.out_iter_sendpage = zerocopy;  in set_out_bvec()
281 	iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,  in set_out_bvec()
282 		      con->v2.out_bvec.bv_len);  in set_out_bvec()
285 static void set_out_bvec_zero(struct ceph_connection *con)  in set_out_bvec_zero()  argument
287 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in set_out_bvec_zero()
288 	WARN_ON(!con->v2.out_zero);  in set_out_bvec_zero()
290 	bvec_set_page(&con->v2.out_bvec, ceph_zero_page,  in set_out_bvec_zero()
291 		      min(con->v2.out_zero, (int)PAGE_SIZE), 0);  in set_out_bvec_zero()
292 	con->v2.out_iter_sendpage = true;  in set_out_bvec_zero()
293 	iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,  in set_out_bvec_zero()
294 		      con->v2.out_bvec.bv_len);  in set_out_bvec_zero()
297 static void out_zero_add(struct ceph_connection *con, int len)  in out_zero_add()  argument
299 	dout("%s con %p len %d\n", __func__, con, len);  in out_zero_add()
300 	con->v2.out_zero += len;  in out_zero_add()
303 static void *alloc_conn_buf(struct ceph_connection *con, int len)  in alloc_conn_buf()  argument
307 	dout("%s con %p len %d\n", __func__, con, len);  in alloc_conn_buf()
309 	if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))  in alloc_conn_buf()
316 	con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;  in alloc_conn_buf()
320 static void free_conn_bufs(struct ceph_connection *con)  in free_conn_bufs()  argument
322 	while (con->v2.conn_buf_cnt)  in free_conn_bufs()
323 		kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);  in free_conn_bufs()
326 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)  in add_in_sign_kvec()  argument
328 	BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));  in add_in_sign_kvec()
330 	con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;  in add_in_sign_kvec()
331 	con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;  in add_in_sign_kvec()
332 	con->v2.in_sign_kvec_cnt++;  in add_in_sign_kvec()
335 static void clear_in_sign_kvecs(struct ceph_connection *con)  in clear_in_sign_kvecs()  argument
337 	con->v2.in_sign_kvec_cnt = 0;  in clear_in_sign_kvecs()
340 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)  in add_out_sign_kvec()  argument
342 	BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));  in add_out_sign_kvec()
344 	con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;  in add_out_sign_kvec()
345 	con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;  in add_out_sign_kvec()
346 	con->v2.out_sign_kvec_cnt++;  in add_out_sign_kvec()
349 static void clear_out_sign_kvecs(struct ceph_connection *con)  in clear_out_sign_kvecs()  argument
351 	con->v2.out_sign_kvec_cnt = 0;  in clear_out_sign_kvecs()
354 static bool con_secure(struct ceph_connection *con)  in con_secure()  argument
356 	return con->v2.con_mode == CEPH_CON_MODE_SECURE;  in con_secure()
565 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)  in encode_epilogue_plain()  argument
567 	con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :  in encode_epilogue_plain()
569 	cpu_to_le32s(&con->v2.out_epil.front_crc);  in encode_epilogue_plain()
570 	cpu_to_le32s(&con->v2.out_epil.middle_crc);  in encode_epilogue_plain()
571 	cpu_to_le32s(&con->v2.out_epil.data_crc);  in encode_epilogue_plain()
574 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)  in encode_epilogue_secure()  argument
576 	memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));  in encode_epilogue_secure()
577 	con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :  in encode_epilogue_secure()
639 static int verify_control_crc(struct ceph_connection *con)  in verify_control_crc()  argument
641 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in verify_control_crc()
644 	WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);  in verify_control_crc()
645 	WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);  in verify_control_crc()
647 	crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);  in verify_control_crc()
648 	expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);  in verify_control_crc()
658 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,  in verify_epilogue_crcs()  argument
661 	if (front_len(con->in_msg)) {  in verify_epilogue_crcs()
662 		con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,  in verify_epilogue_crcs()
663 					   front_len(con->in_msg));  in verify_epilogue_crcs()
665 		WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));  in verify_epilogue_crcs()
666 		con->in_front_crc = -1;  in verify_epilogue_crcs()
669 	if (middle_len(con->in_msg))  in verify_epilogue_crcs()
670 		con->in_middle_crc = crc32c(-1,  in verify_epilogue_crcs()
671 					    con->in_msg->middle->vec.iov_base,  in verify_epilogue_crcs()
672 					    middle_len(con->in_msg));  in verify_epilogue_crcs()
673 	else if (data_len(con->in_msg))  in verify_epilogue_crcs()
674 		con->in_middle_crc = -1;  in verify_epilogue_crcs()
676 		con->in_middle_crc = 0;  in verify_epilogue_crcs()
678 	if (!data_len(con->in_msg))  in verify_epilogue_crcs()
679 		con->in_data_crc = 0;  in verify_epilogue_crcs()
681 	dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,  in verify_epilogue_crcs()
682 	     con->in_front_crc, con->in_middle_crc, con->in_data_crc);  in verify_epilogue_crcs()
684 	if (con->in_front_crc != front_crc) {  in verify_epilogue_crcs()
686 		       con->in_front_crc, front_crc);  in verify_epilogue_crcs()
689 	if (con->in_middle_crc != middle_crc) {  in verify_epilogue_crcs()
691 		       con->in_middle_crc, middle_crc);  in verify_epilogue_crcs()
694 	if (con->in_data_crc != data_crc) {  in verify_epilogue_crcs()
696 		       con->in_data_crc, data_crc);  in verify_epilogue_crcs()
703 static int setup_crypto(struct ceph_connection *con,  in setup_crypto()  argument
710 	dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",  in setup_crypto()
711 	     __func__, con, con->v2.con_mode, session_key_len, con_secret_len);  in setup_crypto()
712 	WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);  in setup_crypto()
714 	if (con->v2.con_mode != CEPH_CON_MODE_CRC &&  in setup_crypto()
715 	    con->v2.con_mode != CEPH_CON_MODE_SECURE) {  in setup_crypto()
716 		pr_err("bad con_mode %d\n", con->v2.con_mode);  in setup_crypto()
721 		WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);  in setup_crypto()
727 	con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);  in setup_crypto()
729 	if (IS_ERR(con->v2.hmac_tfm)) {  in setup_crypto()
730 		ret = PTR_ERR(con->v2.hmac_tfm);  in setup_crypto()
731 		con->v2.hmac_tfm = NULL;  in setup_crypto()
736 	ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,  in setup_crypto()
743 	if (con->v2.con_mode == CEPH_CON_MODE_CRC) {  in setup_crypto()
754 	con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);  in setup_crypto()
756 	if (IS_ERR(con->v2.gcm_tfm)) {  in setup_crypto()
757 		ret = PTR_ERR(con->v2.gcm_tfm);  in setup_crypto()
758 		con->v2.gcm_tfm = NULL;  in setup_crypto()
764 		crypto_aead_alignmask(con->v2.gcm_tfm));  in setup_crypto()
765 	ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);  in setup_crypto()
771 	WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);  in setup_crypto()
772 	ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);  in setup_crypto()
778 	con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);  in setup_crypto()
779 	if (!con->v2.gcm_req) {  in setup_crypto()
784 	crypto_init_wait(&con->v2.gcm_wait);  in setup_crypto()
785 	aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in setup_crypto()
786 				  crypto_req_done, &con->v2.gcm_wait);  in setup_crypto()
788 	memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,  in setup_crypto()
790 	memcpy(&con->v2.out_gcm_nonce,  in setup_crypto()
796 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,  in hmac_sha256()  argument
799 	SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm);  /* tfm arg is ignored */  in hmac_sha256()
803 	dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,  in hmac_sha256()
804 	     con->v2.hmac_tfm, kvec_cnt);  in hmac_sha256()
806 	if (!con->v2.hmac_tfm) {  in hmac_sha256()
811 	desc->tfm = con->v2.hmac_tfm;  in hmac_sha256()
838 static int gcm_crypt(struct ceph_connection *con, bool encrypt,  in gcm_crypt()  argument
845 	nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;  in gcm_crypt()
847 	aead_request_set_ad(con->v2.gcm_req, 0);  /* no AAD */  in gcm_crypt()
848 	aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);  in gcm_crypt()
849 	ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :  in gcm_crypt()
850 					crypto_aead_decrypt(con->v2.gcm_req),  in gcm_crypt()
851 			      &con->v2.gcm_wait);  in gcm_crypt()
1060 static int decrypt_preamble(struct ceph_connection *con)  in decrypt_preamble()  argument
1064 	sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);  in decrypt_preamble()
1065 	return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);  in decrypt_preamble()
1068 static int decrypt_control_remainder(struct ceph_connection *con)  in decrypt_control_remainder()  argument
1070 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in decrypt_control_remainder()
1075 	WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);  in decrypt_control_remainder()
1076 	WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);  in decrypt_control_remainder()
1079 	sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);  in decrypt_control_remainder()
1080 	sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);  in decrypt_control_remainder()
1082 	return gcm_crypt(con, false, sgs, sgs,  in decrypt_control_remainder()
1087 static int process_v2_sparse_read(struct ceph_connection *con,  in process_v2_sparse_read()  argument
1090 	struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;  in process_v2_sparse_read()
1096 		ret = con->ops->sparse_read(con, cursor, &buf);  in process_v2_sparse_read()
1105 			struct page *spage = con->v2.in_enc_pages[idx];  in process_v2_sparse_read()
1126 static int decrypt_tail(struct ceph_connection *con)  in decrypt_tail()  argument
1131 	bool sparse = !!con->in_msg->sparse_read_total;  in decrypt_tail()
1136 	tail_len = tail_onwire_len(con->in_msg, true);  in decrypt_tail()
1137 	ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,  in decrypt_tail()
1138 					con->v2.in_enc_page_cnt, 0, tail_len,  in decrypt_tail()
1144 		dpos = padded_len(front_len(con->in_msg) + padded_len(middle_len(con->in_msg)));  in decrypt_tail()
1145 		pages = con->v2.in_enc_pages;  in decrypt_tail()
1148 	ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),  in decrypt_tail()
1149 				MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),  in decrypt_tail()
1150 				con->v2.in_buf, pages, dpos, true);  in decrypt_tail()
1154 	dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,  in decrypt_tail()
1155 	     con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);  in decrypt_tail()
1156 	ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);  in decrypt_tail()
1160 	if (sparse && data_len(con->in_msg)) {  in decrypt_tail()
1161 		ret = process_v2_sparse_read(con, con->v2.in_enc_pages, dpos);  in decrypt_tail()
1166 	WARN_ON(!con->v2.in_enc_page_cnt);  in decrypt_tail()
1167 	ceph_release_page_vector(con->v2.in_enc_pages,  in decrypt_tail()
1168 				 con->v2.in_enc_page_cnt);  in decrypt_tail()
1169 	con->v2.in_enc_pages = NULL;  in decrypt_tail()
1170 	con->v2.in_enc_page_cnt = 0;  in decrypt_tail()
1178 static int prepare_banner(struct ceph_connection *con)  in prepare_banner()  argument
1183 	buf = alloc_conn_buf(con, buf_len);  in prepare_banner()
1194 	add_out_kvec(con, buf, buf_len);  in prepare_banner()
1195 	add_out_sign_kvec(con, buf, buf_len);  in prepare_banner()
1196 	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);  in prepare_banner()
1217 static void prepare_head_plain(struct ceph_connection *con, void *base,  in prepare_head_plain()  argument
1231 		add_out_kvec(con, base, base_len);  in prepare_head_plain()
1233 			add_out_sign_kvec(con, base, base_len);  in prepare_head_plain()
1237 	add_out_kvec(con, base, crcp - base);  in prepare_head_plain()
1238 	add_out_kvec(con, extdata, extdata_len);  in prepare_head_plain()
1239 	add_out_kvec(con, crcp, CEPH_CRC_LEN);  in prepare_head_plain()
1241 		add_out_sign_kvec(con, base, crcp - base);  in prepare_head_plain()
1242 		add_out_sign_kvec(con, extdata, extdata_len);  in prepare_head_plain()
1243 		add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);  in prepare_head_plain()
1247 static int prepare_head_secure_small(struct ceph_connection *con,  in prepare_head_secure_small()  argument
1259 	ret = gcm_crypt(con, true, &sg, &sg,  in prepare_head_secure_small()
1264 	add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);  in prepare_head_secure_small()
1288 static int prepare_head_secure_big(struct ceph_connection *con,  in prepare_head_secure_big()  argument
1301 	ret = gcm_crypt(con, true, sgs, sgs, rem - base);  in prepare_head_secure_big()
1310 	ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);  in prepare_head_secure_big()
1314 	add_out_kvec(con, base, rem - base);  in prepare_head_secure_big()
1315 	add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);  in prepare_head_secure_big()
1316 	add_out_kvec(con, rem, pmbl_tag - rem);  in prepare_head_secure_big()
1320 static int __prepare_control(struct ceph_connection *con, int tag,  in __prepare_control()  argument
1328 	dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,  in __prepare_control()
1338 	if (con_secure(con)) {  in __prepare_control()
1344 			ret = prepare_head_secure_small(con, base, ctrl_len);  in __prepare_control()
1347 			ret = prepare_head_secure_big(con, base, ctrl_len);  in __prepare_control()
1351 		prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,  in __prepare_control()
1355 	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);  in __prepare_control()
1359 static int prepare_control(struct ceph_connection *con, int tag,  in prepare_control()  argument
1362 	return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);  in prepare_control()
1365 static int prepare_hello(struct ceph_connection *con)  in prepare_hello()  argument
1370 	ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);  in prepare_hello()
1371 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));  in prepare_hello()
1377 	ceph_encode_entity_addr(&p, &con->peer_addr);  in prepare_hello()
1380 	return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,  in prepare_hello()
1387 static int prepare_auth_request(struct ceph_connection *con)  in prepare_auth_request()  argument
1395 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));  in prepare_auth_request()
1399 	mutex_unlock(&con->mutex);  in prepare_auth_request()
1400 	ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,  in prepare_auth_request()
1402 	mutex_lock(&con->mutex);  in prepare_auth_request()
1403 	if (con->state != CEPH_CON_S_V2_HELLO) {  in prepare_auth_request()
1404 		dout("%s con %p state changed to %d\n", __func__, con,  in prepare_auth_request()
1405 		     con->state);  in prepare_auth_request()
1409 	dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);  in prepare_auth_request()
1413 	authorizer_copy = alloc_conn_buf(con, authorizer_len);  in prepare_auth_request()
1419 	return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,  in prepare_auth_request()
1423 static int prepare_auth_request_more(struct ceph_connection *con,  in prepare_auth_request_more()  argument
1432 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));  in prepare_auth_request_more()
1436 	mutex_unlock(&con->mutex);  in prepare_auth_request_more()
1437 	ret = con->ops->handle_auth_reply_more(con, reply, reply_len,  in prepare_auth_request_more()
1440 	mutex_lock(&con->mutex);  in prepare_auth_request_more()
1441 	if (con->state != CEPH_CON_S_V2_AUTH) {  in prepare_auth_request_more()
1442 		dout("%s con %p state changed to %d\n", __func__, con,  in prepare_auth_request_more()
1443 		     con->state);  in prepare_auth_request_more()
1447 	dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);  in prepare_auth_request_more()
1451 	return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,  in prepare_auth_request_more()
1455 static int prepare_auth_signature(struct ceph_connection *con)  in prepare_auth_signature()  argument
1460 	buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,  in prepare_auth_signature()
1461 						  con_secure(con)));  in prepare_auth_signature()
1465 	ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,  in prepare_auth_signature()
1470 	return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,  in prepare_auth_signature()
1474 static int prepare_client_ident(struct ceph_connection *con)  in prepare_client_ident()  argument
1476 	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;  in prepare_client_ident()
1477 	struct ceph_client *client = from_msgr(con->msgr);  in prepare_client_ident()
1482 	WARN_ON(con->v2.server_cookie);  in prepare_client_ident()
1483 	WARN_ON(con->v2.connect_seq);  in prepare_client_ident()
1484 	WARN_ON(con->v2.peer_global_seq);  in prepare_client_ident()
1486 	if (!con->v2.client_cookie) {  in prepare_client_ident()
1488 			get_random_bytes(&con->v2.client_cookie,  in prepare_client_ident()
1489 					 sizeof(con->v2.client_cookie));  in prepare_client_ident()
1490 		} while (!con->v2.client_cookie);  in prepare_client_ident()
1491 		dout("%s con %p generated cookie 0x%llx\n", __func__, con,  in prepare_client_ident()
1492 		     con->v2.client_cookie);  in prepare_client_ident()
1494 		dout("%s con %p cookie already set 0x%llx\n", __func__, con,  in prepare_client_ident()
1495 		     con->v2.client_cookie);  in prepare_client_ident()
1498 …dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx requi…  in prepare_client_ident()
1499 	     __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),  in prepare_client_ident()
1500 	     ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),  in prepare_client_ident()
1501 	     global_id, con->v2.global_seq, client->supported_features,  in prepare_client_ident()
1502 	     client->required_features, con->v2.client_cookie);  in prepare_client_ident()
1505 		   ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;  in prepare_client_ident()
1506 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));  in prepare_client_ident()
1514 	ceph_encode_entity_addr(&p, &con->peer_addr);  in prepare_client_ident()
1516 	ceph_encode_64(&p, con->v2.global_seq);  in prepare_client_ident()
1520 	ceph_encode_64(&p, con->v2.client_cookie);  in prepare_client_ident()
1523 	return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);  in prepare_client_ident()
1526 static int prepare_session_reconnect(struct ceph_connection *con)  in prepare_session_reconnect()  argument
1528 	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;  in prepare_session_reconnect()
1532 	WARN_ON(!con->v2.client_cookie);  in prepare_session_reconnect()
1533 	WARN_ON(!con->v2.server_cookie);  in prepare_session_reconnect()
1534 	WARN_ON(!con->v2.connect_seq);  in prepare_session_reconnect()
1535 	WARN_ON(!con->v2.peer_global_seq);  in prepare_session_reconnect()
1537 …dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_se…  in prepare_session_reconnect()
1538 	     __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),  in prepare_session_reconnect()
1539 	     con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,  in prepare_session_reconnect()
1540 	     con->v2.connect_seq, con->in_seq);  in prepare_session_reconnect()
1543 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));  in prepare_session_reconnect()
1551 	ceph_encode_64(&p, con->v2.client_cookie);  in prepare_session_reconnect()
1552 	ceph_encode_64(&p, con->v2.server_cookie);  in prepare_session_reconnect()
1553 	ceph_encode_64(&p, con->v2.global_seq);  in prepare_session_reconnect()
1554 	ceph_encode_64(&p, con->v2.connect_seq);  in prepare_session_reconnect()
1555 	ceph_encode_64(&p, con->in_seq);  in prepare_session_reconnect()
1558 	return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);  in prepare_session_reconnect()
1561 static int prepare_keepalive2(struct ceph_connection *con)  in prepare_keepalive2()  argument
1563 	struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);  in prepare_keepalive2()
1567 	dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,  in prepare_keepalive2()
1572 	reset_out_kvecs(con);  in prepare_keepalive2()
1573 	return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,  in prepare_keepalive2()
1577 static int prepare_ack(struct ceph_connection *con)  in prepare_ack()  argument
1581 	dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,  in prepare_ack()
1582 	     con->in_seq_acked, con->in_seq);  in prepare_ack()
1583 	con->in_seq_acked = con->in_seq;  in prepare_ack()
1585 	p = CTRL_BODY(con->v2.out_buf);  in prepare_ack()
1586 	ceph_encode_64(&p, con->in_seq_acked);  in prepare_ack()
1588 	reset_out_kvecs(con);  in prepare_ack()
1589 	return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);  in prepare_ack()
1592 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)  in prepare_epilogue_plain()  argument
1594 	dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,  in prepare_epilogue_plain()
1595 	     con->out_msg, aborted, con->v2.out_epil.front_crc,  in prepare_epilogue_plain()
1596 	     con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);  in prepare_epilogue_plain()
1598 	encode_epilogue_plain(con, aborted);  in prepare_epilogue_plain()
1599 	add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_epilogue_plain()
1606 static void prepare_message_plain(struct ceph_connection *con)  in prepare_message_plain()  argument
1608 	struct ceph_msg *msg = con->out_msg;  in prepare_message_plain()
1610 	prepare_head_plain(con, con->v2.out_buf,  in prepare_message_plain()
1619 			con->v2.out_state = OUT_S_FINISH_MESSAGE;  in prepare_message_plain()
1623 		con->v2.out_epil.front_crc = -1;  in prepare_message_plain()
1624 		con->v2.out_epil.middle_crc = -1;  in prepare_message_plain()
1625 		con->v2.out_state = OUT_S_QUEUE_DATA;  in prepare_message_plain()
1630 		con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,  in prepare_message_plain()
1632 		add_out_kvec(con, msg->front.iov_base, front_len(msg));  in prepare_message_plain()
1635 		con->v2.out_epil.front_crc = -1;  in prepare_message_plain()
1639 		con->v2.out_epil.middle_crc =  in prepare_message_plain()
1641 		add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));  in prepare_message_plain()
1643 		con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;  in prepare_message_plain()
1647 		con->v2.out_state = OUT_S_QUEUE_DATA;  in prepare_message_plain()
1649 		con->v2.out_epil.data_crc = 0;  in prepare_message_plain()
1650 		prepare_epilogue_plain(con, false);  in prepare_message_plain()
1651 		con->v2.out_state = OUT_S_FINISH_MESSAGE;  in prepare_message_plain()
1662 static int prepare_message_secure(struct ceph_connection *con)  in prepare_message_secure()  argument
1672 	ret = prepare_head_secure_small(con, con->v2.out_buf,  in prepare_message_secure()
1677 	tail_len = tail_onwire_len(con->out_msg, true);  in prepare_message_secure()
1683 		con->v2.out_state = OUT_S_FINISH_MESSAGE;  in prepare_message_secure()
1687 	encode_epilogue_secure(con, false);  in prepare_message_secure()
1688 	ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,  in prepare_message_secure()
1689 				&con->v2.out_epil, NULL, 0, false);  in prepare_message_secure()
1700 	WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);  in prepare_message_secure()
1701 	con->v2.out_enc_pages = enc_pages;  in prepare_message_secure()
1702 	con->v2.out_enc_page_cnt = enc_page_cnt;  in prepare_message_secure()
1703 	con->v2.out_enc_resid = tail_len;  in prepare_message_secure()
1704 	con->v2.out_enc_i = 0;  in prepare_message_secure()
1711 	ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,  in prepare_message_secure()
1716 	dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,  in prepare_message_secure()
1717 	     con->out_msg, sgt.orig_nents, enc_page_cnt);  in prepare_message_secure()
1718 	con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;  in prepare_message_secure()
1726 static int prepare_message(struct ceph_connection *con)  in prepare_message()  argument
1730 		front_len(con->out_msg),  in prepare_message()
1731 		middle_len(con->out_msg),  in prepare_message()
1732 		data_len(con->out_msg)  in prepare_message()
1737 	dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,  in prepare_message()
1738 	     con->out_msg, lens[0], lens[1], lens[2], lens[3]);  in prepare_message()
1740 	if (con->in_seq > con->in_seq_acked) {  in prepare_message()
1741 		dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,  in prepare_message()
1742 		     con->in_seq_acked, con->in_seq);  in prepare_message()
1743 		con->in_seq_acked = con->in_seq;  in prepare_message()
1746 	reset_out_kvecs(con);  in prepare_message()
1748 	encode_preamble(&desc, con->v2.out_buf);  in prepare_message()
1749 	fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,  in prepare_message()
1750 		     con->in_seq_acked);  in prepare_message()
1752 	if (con_secure(con)) {  in prepare_message()
1753 		ret = prepare_message_secure(con);  in prepare_message()
1757 		prepare_message_plain(con);  in prepare_message()
1760 	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);  in prepare_message()
1764 static int prepare_read_banner_prefix(struct ceph_connection *con)  in prepare_read_banner_prefix()  argument
1768 	buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);  in prepare_read_banner_prefix()
1772 	reset_in_kvecs(con);  in prepare_read_banner_prefix()
1773 	add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);  in prepare_read_banner_prefix()
1774 	add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);  in prepare_read_banner_prefix()
1775 	con->state = CEPH_CON_S_V2_BANNER_PREFIX;  in prepare_read_banner_prefix()
1779 static int prepare_read_banner_payload(struct ceph_connection *con,  in prepare_read_banner_payload()  argument
1784 	buf = alloc_conn_buf(con, payload_len);  in prepare_read_banner_payload()
1788 	reset_in_kvecs(con);  in prepare_read_banner_payload()
1789 	add_in_kvec(con, buf, payload_len);  in prepare_read_banner_payload()
1790 	add_in_sign_kvec(con, buf, payload_len);  in prepare_read_banner_payload()
1791 	con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;  in prepare_read_banner_payload()
1795 static void prepare_read_preamble(struct ceph_connection *con)  in prepare_read_preamble()  argument
1797 	reset_in_kvecs(con);  in prepare_read_preamble()
1798 	add_in_kvec(con, con->v2.in_buf,  in prepare_read_preamble()
1799 		    con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :  in prepare_read_preamble()
1801 	con->v2.in_state = IN_S_HANDLE_PREAMBLE;  in prepare_read_preamble()
1804 static int prepare_read_control(struct ceph_connection *con)  in prepare_read_control()  argument
1806 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in prepare_read_control()
1810 	reset_in_kvecs(con);  in prepare_read_control()
1811 	if (con->state == CEPH_CON_S_V2_HELLO ||  in prepare_read_control()
1812 	    con->state == CEPH_CON_S_V2_AUTH) {  in prepare_read_control()
1814 		buf = alloc_conn_buf(con, head_len);  in prepare_read_control()
1819 		memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);  in prepare_read_control()
1821 		add_in_kvec(con, CTRL_BODY(buf), ctrl_len);  in prepare_read_control()
1822 		add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);  in prepare_read_control()
1823 		add_in_sign_kvec(con, buf, head_len);  in prepare_read_control()
1826 			buf = alloc_conn_buf(con, ctrl_len);  in prepare_read_control()
1830 			add_in_kvec(con, buf, ctrl_len);  in prepare_read_control()
1832 			add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);  in prepare_read_control()
1834 		add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);  in prepare_read_control()
1836 	con->v2.in_state = IN_S_HANDLE_CONTROL;  in prepare_read_control()
1840 static int prepare_read_control_remainder(struct ceph_connection *con)  in prepare_read_control_remainder()  argument
1842 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in prepare_read_control_remainder()
1846 	buf = alloc_conn_buf(con, ctrl_len);  in prepare_read_control_remainder()
1850 	memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);  in prepare_read_control_remainder()
1852 	reset_in_kvecs(con);  in prepare_read_control_remainder()
1853 	add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);  in prepare_read_control_remainder()
1854 	add_in_kvec(con, con->v2.in_buf,  in prepare_read_control_remainder()
1856 	con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;  in prepare_read_control_remainder()
1860 static int prepare_read_data(struct ceph_connection *con)  in prepare_read_data()  argument
1864 	con->in_data_crc = -1;  in prepare_read_data()
1865 	ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,  in prepare_read_data()
1866 				  data_len(con->in_msg));  in prepare_read_data()
1868 	get_bvec_at(&con->v2.in_cursor, &bv);  in prepare_read_data()
1869 	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_read_data()
1870 		if (unlikely(!con->bounce_page)) {  in prepare_read_data()
1871 			con->bounce_page = alloc_page(GFP_NOIO);  in prepare_read_data()
1872 			if (!con->bounce_page) {  in prepare_read_data()
1878 		bv.bv_page = con->bounce_page;  in prepare_read_data()
1881 	set_in_bvec(con, &bv);  in prepare_read_data()
1882 	con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;  in prepare_read_data()
1886 static void prepare_read_data_cont(struct ceph_connection *con)  in prepare_read_data_cont()  argument
1890 	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_read_data_cont()
1891 		con->in_data_crc = crc32c(con->in_data_crc,  in prepare_read_data_cont()
1892 					  page_address(con->bounce_page),  in prepare_read_data_cont()
1893 					  con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1895 		get_bvec_at(&con->v2.in_cursor, &bv);  in prepare_read_data_cont()
1897 			       page_address(con->bounce_page),  in prepare_read_data_cont()
1898 			       con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1900 		con->in_data_crc = ceph_crc32c_page(con->in_data_crc,  in prepare_read_data_cont()
1901 						    con->v2.in_bvec.bv_page,  in prepare_read_data_cont()
1902 						    con->v2.in_bvec.bv_offset,  in prepare_read_data_cont()
1903 						    con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1906 	ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1907 	if (con->v2.in_cursor.total_resid) {  in prepare_read_data_cont()
1908 		get_bvec_at(&con->v2.in_cursor, &bv);  in prepare_read_data_cont()
1909 		if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_read_data_cont()
1910 			bv.bv_page = con->bounce_page;  in prepare_read_data_cont()
1913 		set_in_bvec(con, &bv);  in prepare_read_data_cont()
1914 		WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);  in prepare_read_data_cont()
1921 	reset_in_kvecs(con);  in prepare_read_data_cont()
1922 	add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_read_data_cont()
1923 	con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_read_data_cont()
1926 static int prepare_sparse_read_cont(struct ceph_connection *con)  in prepare_sparse_read_cont()  argument
1931 	struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;  in prepare_sparse_read_cont()
1933 	WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT);  in prepare_sparse_read_cont()
1935 	if (iov_iter_is_bvec(&con->v2.in_iter)) {  in prepare_sparse_read_cont()
1936 		if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_sparse_read_cont()
1937 			con->in_data_crc = crc32c(con->in_data_crc,  in prepare_sparse_read_cont()
1938 						  page_address(con->bounce_page),  in prepare_sparse_read_cont()
1939 						  con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1942 				       page_address(con->bounce_page),  in prepare_sparse_read_cont()
1943 				       con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1945 			con->in_data_crc = ceph_crc32c_page(con->in_data_crc,  in prepare_sparse_read_cont()
1946 							    con->v2.in_bvec.bv_page,  in prepare_sparse_read_cont()
1947 							    con->v2.in_bvec.bv_offset,  in prepare_sparse_read_cont()
1948 							    con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1951 		ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1952 		cursor->sr_resid -= con->v2.in_bvec.bv_len;  in prepare_sparse_read_cont()
1954 		     con->v2.in_bvec.bv_len, cursor->sr_resid);  in prepare_sparse_read_cont()
1960 			if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_sparse_read_cont()
1961 				bv.bv_page = con->bounce_page;  in prepare_sparse_read_cont()
1964 			set_in_bvec(con, &bv);  in prepare_sparse_read_cont()
1965 			con->v2.data_len_remain -= bv.bv_len;  in prepare_sparse_read_cont()
1968 	} else if (iov_iter_is_kvec(&con->v2.in_iter)) {  in prepare_sparse_read_cont()
1970 		if (con->v2.in_kvec_cnt) {  in prepare_sparse_read_cont()
1971 			WARN_ON_ONCE(con->v2.in_kvec_cnt > 1);  in prepare_sparse_read_cont()
1972 			con->in_data_crc = crc32c(con->in_data_crc,  in prepare_sparse_read_cont()
1973 						  con->v2.in_kvecs[0].iov_base,  in prepare_sparse_read_cont()
1974 						  con->v2.in_kvecs[0].iov_len);  in prepare_sparse_read_cont()
1981 	ret = con->ops->sparse_read(con, cursor, &buf);  in prepare_sparse_read_cont()
1986 		reset_in_kvecs(con);  in prepare_sparse_read_cont()
1987 		add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_sparse_read_cont()
1988 		con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_sparse_read_cont()
1994 		reset_in_kvecs(con);  in prepare_sparse_read_cont()
1995 		add_in_kvec(con, buf, ret);  in prepare_sparse_read_cont()
1996 		con->v2.data_len_remain -= ret;  in prepare_sparse_read_cont()
2008 	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_sparse_read_cont()
2009 		if (unlikely(!con->bounce_page)) {  in prepare_sparse_read_cont()
2010 			con->bounce_page = alloc_page(GFP_NOIO);  in prepare_sparse_read_cont()
2011 			if (!con->bounce_page) {  in prepare_sparse_read_cont()
2017 		bv.bv_page = con->bounce_page;  in prepare_sparse_read_cont()
2020 	set_in_bvec(con, &bv);  in prepare_sparse_read_cont()
2021 	con->v2.data_len_remain -= ret;  in prepare_sparse_read_cont()
2025 static int prepare_sparse_read_data(struct ceph_connection *con)  in prepare_sparse_read_data()  argument
2027 	struct ceph_msg *msg = con->in_msg;  in prepare_sparse_read_data()
2031 	if (WARN_ON_ONCE(!con->ops->sparse_read))  in prepare_sparse_read_data()
2034 	if (!con_secure(con))  in prepare_sparse_read_data()
2035 		con->in_data_crc = -1;  in prepare_sparse_read_data()
2037 	ceph_msg_data_cursor_init(&con->v2.in_cursor, msg,  in prepare_sparse_read_data()
2040 	reset_in_kvecs(con);  in prepare_sparse_read_data()
2041 	con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT;  in prepare_sparse_read_data()
2042 	con->v2.data_len_remain = data_len(msg);  in prepare_sparse_read_data()
2043 	return prepare_sparse_read_cont(con);  in prepare_sparse_read_data()
2046 static int prepare_read_tail_plain(struct ceph_connection *con)  in prepare_read_tail_plain()  argument
2048 	struct ceph_msg *msg = con->in_msg;  in prepare_read_tail_plain()
2052 		return prepare_read_data(con);  in prepare_read_tail_plain()
2055 	reset_in_kvecs(con);  in prepare_read_tail_plain()
2057 		add_in_kvec(con, msg->front.iov_base, front_len(msg));  in prepare_read_tail_plain()
2061 		add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));  in prepare_read_tail_plain()
2067 			con->v2.in_state = IN_S_PREPARE_SPARSE_DATA;  in prepare_read_tail_plain()
2069 			con->v2.in_state = IN_S_PREPARE_READ_DATA;  in prepare_read_tail_plain()
2071 		add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_read_tail_plain()
2072 		con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_read_tail_plain()
2077 static void prepare_read_enc_page(struct ceph_connection *con)  in prepare_read_enc_page()  argument
2081 	dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,  in prepare_read_enc_page()
2082 	     con->v2.in_enc_resid);  in prepare_read_enc_page()
2083 	WARN_ON(!con->v2.in_enc_resid);  in prepare_read_enc_page()
2085 	bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i],  in prepare_read_enc_page()
2086 		      min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0);  in prepare_read_enc_page()
2088 	set_in_bvec(con, &bv);  in prepare_read_enc_page()
2089 	con->v2.in_enc_i++;  in prepare_read_enc_page()
2090 	con->v2.in_enc_resid -= bv.bv_len;  in prepare_read_enc_page()
2092 	if (con->v2.in_enc_resid) {  in prepare_read_enc_page()
2093 		con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;  in prepare_read_enc_page()
2101 	WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);  in prepare_read_enc_page()
2102 	con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_read_enc_page()
2105 static int prepare_read_tail_secure(struct ceph_connection *con)  in prepare_read_tail_secure()  argument
2111 	tail_len = tail_onwire_len(con->in_msg, true);  in prepare_read_tail_secure()
2119 	WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);  in prepare_read_tail_secure()
2120 	con->v2.in_enc_pages = enc_pages;  in prepare_read_tail_secure()
2121 	con->v2.in_enc_page_cnt = enc_page_cnt;  in prepare_read_tail_secure()
2122 	con->v2.in_enc_resid = tail_len;  in prepare_read_tail_secure()
2123 	con->v2.in_enc_i = 0;  in prepare_read_tail_secure()
2125 	prepare_read_enc_page(con);  in prepare_read_tail_secure()
2129 static void __finish_skip(struct ceph_connection *con)  in __finish_skip()  argument
2131 	con->in_seq++;  in __finish_skip()
2132 	prepare_read_preamble(con);  in __finish_skip()
2135 static void prepare_skip_message(struct ceph_connection *con)  in prepare_skip_message()  argument
2137 	struct ceph_frame_desc *desc = &con->v2.in_desc;  in prepare_skip_message()
2140 	dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],  in prepare_skip_message()
2144 				     desc->fd_lens[3], con_secure(con));  in prepare_skip_message()
2146 		__finish_skip(con);  in prepare_skip_message()
2148 		set_in_skip(con, tail_len);  in prepare_skip_message()
2149 		con->v2.in_state = IN_S_FINISH_SKIP;  in prepare_skip_message()
2153 static int process_banner_prefix(struct ceph_connection *con)  in process_banner_prefix()  argument
2158 	WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);  in process_banner_prefix()
2160 	p = con->v2.in_kvecs[0].iov_base;  in process_banner_prefix()
2163 			con->error_msg = "server is speaking msgr1 protocol";  in process_banner_prefix()
2165 			con->error_msg = "protocol error, bad banner";  in process_banner_prefix()
2171 	dout("%s con %p payload_len %d\n", __func__, con, payload_len);  in process_banner_prefix()
2173 	return prepare_read_banner_payload(con, payload_len);  in process_banner_prefix()
2176 static int process_banner_payload(struct ceph_connection *con)  in process_banner_payload()  argument
2178 	void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;  in process_banner_payload()
2185 	p = con->v2.in_kvecs[0].iov_base;  in process_banner_payload()
2189 	dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",  in process_banner_payload()
2190 	     __func__, con, server_feat, server_req_feat);  in process_banner_payload()
2195 		con->error_msg = "missing required protocol features";  in process_banner_payload()
2201 		con->error_msg = "missing required protocol features";  in process_banner_payload()
2206 	ret = prepare_hello(con);  in process_banner_payload()
2212 	con->state = CEPH_CON_S_V2_HELLO;  in process_banner_payload()
2213 	prepare_read_preamble(con);  in process_banner_payload()
2221 static int process_hello(struct ceph_connection *con, void *p, void *end)  in process_hello()  argument
2223 	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;  in process_hello()
2228 	if (con->state != CEPH_CON_S_V2_HELLO) {  in process_hello()
2229 		con->error_msg = "protocol error, unexpected hello";  in process_hello()
2240 	dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,  in process_hello()
2243 	if (entity_type != con->peer_name.type) {  in process_hello()
2245 		       con->peer_name.type, entity_type);  in process_hello()
2246 		con->error_msg = "wrong peer at address";  in process_hello()
2260 		dout("%s con %p set my addr %s, as seen by peer %s\n",  in process_hello()
2261 		     __func__, con, ceph_pr_addr(my_addr),  in process_hello()
2262 		     ceph_pr_addr(&con->peer_addr));  in process_hello()
2264 		dout("%s con %p my addr already set %s\n",  in process_hello()
2265 		     __func__, con, ceph_pr_addr(my_addr));  in process_hello()
2273 	ret = prepare_auth_request(con);  in process_hello()
2280 	con->state = CEPH_CON_S_V2_AUTH;  in process_hello()
2288 static int process_auth_bad_method(struct ceph_connection *con,  in process_auth_bad_method()  argument
2297 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_bad_method()
2298 		con->error_msg = "protocol error, unexpected auth_bad_method";  in process_auth_bad_method()
2304 	dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,  in process_auth_bad_method()
2314 		dout("%s con %p allowed_protos[%d] %d\n", __func__, con,  in process_auth_bad_method()
2325 		dout("%s con %p allowed_modes[%d] %d\n", __func__, con,  in process_auth_bad_method()
2329 	mutex_unlock(&con->mutex);  in process_auth_bad_method()
2330 	ret = con->ops->handle_auth_bad_method(con, used_proto, result,  in process_auth_bad_method()
2335 	mutex_lock(&con->mutex);  in process_auth_bad_method()
2336 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_bad_method()
2337 		dout("%s con %p state changed to %d\n", __func__, con,  in process_auth_bad_method()
2338 		     con->state);  in process_auth_bad_method()
2342 	dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);  in process_auth_bad_method()
2350 static int process_auth_reply_more(struct ceph_connection *con,  in process_auth_reply_more()  argument
2356 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_reply_more()
2357 		con->error_msg = "protocol error, unexpected auth_reply_more";  in process_auth_reply_more()
2364 	dout("%s con %p payload_len %d\n", __func__, con, payload_len);  in process_auth_reply_more()
2366 	reset_out_kvecs(con);  in process_auth_reply_more()
2367 	ret = prepare_auth_request_more(con, p, payload_len);  in process_auth_reply_more()
2387 static int process_auth_done(struct ceph_connection *con, void *p, void *end)  in process_auth_done()  argument
2398 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_done()
2399 		con->error_msg = "protocol error, unexpected auth_done";  in process_auth_done()
2404 	ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);  in process_auth_done()
2407 	dout("%s con %p global_id %llu con_mode %d payload_len %d\n",  in process_auth_done()
2408 	     __func__, con, global_id, con->v2.con_mode, payload_len);  in process_auth_done()
2410 	mutex_unlock(&con->mutex);  in process_auth_done()
2413 	ret = con->ops->handle_auth_done(con, global_id, p, payload_len,  in process_auth_done()
2416 	mutex_lock(&con->mutex);  in process_auth_done()
2417 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_done()
2418 		dout("%s con %p state changed to %d\n", __func__, con,  in process_auth_done()
2419 		     con->state);  in process_auth_done()
2424 	dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);  in process_auth_done()
2428 	ret = setup_crypto(con, session_key, session_key_len, con_secret,  in process_auth_done()
2433 	reset_out_kvecs(con);  in process_auth_done()
2434 	ret = prepare_auth_signature(con);  in process_auth_done()
2440 	con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;  in process_auth_done()
2452 static int process_auth_signature(struct ceph_connection *con,  in process_auth_signature()  argument
2458 	if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {  in process_auth_signature()
2459 		con->error_msg = "protocol error, unexpected auth_signature";  in process_auth_signature()
2463 	ret = hmac_sha256(con, con->v2.out_sign_kvecs,  in process_auth_signature()
2464 			  con->v2.out_sign_kvec_cnt, hmac);  in process_auth_signature()
2470 		con->error_msg = "integrity error, bad auth signature";  in process_auth_signature()
2474 	dout("%s con %p auth signature ok\n", __func__, con);  in process_auth_signature()
2477 	if (!con->v2.server_cookie) {  in process_auth_signature()
2478 		ret = prepare_client_ident(con);  in process_auth_signature()
2484 		con->state = CEPH_CON_S_V2_SESSION_CONNECT;  in process_auth_signature()
2486 		ret = prepare_session_reconnect(con);  in process_auth_signature()
2492 		con->state = CEPH_CON_S_V2_SESSION_RECONNECT;  in process_auth_signature()
2502 static int process_server_ident(struct ceph_connection *con,  in process_server_ident()  argument
2505 	struct ceph_client *client = from_msgr(con->msgr);  in process_server_ident()
2514 	if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {  in process_server_ident()
2515 		con->error_msg = "protocol error, unexpected server_ident";  in process_server_ident()
2532 …dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx…  in process_server_ident()
2533 	     __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),  in process_server_ident()
2537 	if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {  in process_server_ident()
2539 		       ceph_pr_addr(&con->peer_addr),  in process_server_ident()
2540 		       le32_to_cpu(con->peer_addr.nonce),  in process_server_ident()
2542 		con->error_msg = "wrong peer at address";  in process_server_ident()
2549 		con->error_msg = "missing required protocol features";  in process_server_ident()
2558 	WARN_ON(!con->peer_name.type);  in process_server_ident()
2559 	con->peer_name.num = cpu_to_le64(global_id);  in process_server_ident()
2560 	con->v2.peer_global_seq = global_seq;  in process_server_ident()
2561 	con->peer_features = features;  in process_server_ident()
2563 	con->v2.server_cookie = cookie;  in process_server_ident()
2566 		ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);  in process_server_ident()
2567 		WARN_ON(con->v2.server_cookie);  in process_server_ident()
2569 		WARN_ON(!con->v2.server_cookie);  in process_server_ident()
2572 	clear_in_sign_kvecs(con);  in process_server_ident()
2573 	clear_out_sign_kvecs(con);  in process_server_ident()
2574 	free_conn_bufs(con);  in process_server_ident()
2575 	con->delay = 0;  /* reset backoff memory */  in process_server_ident()
2577 	con->state = CEPH_CON_S_OPEN;  in process_server_ident()
2578 	con->v2.out_state = OUT_S_GET_NEXT;  in process_server_ident()
2586 static int process_ident_missing_features(struct ceph_connection *con,  in process_ident_missing_features()  argument
2589 	struct ceph_client *client = from_msgr(con->msgr);  in process_ident_missing_features()
2592 	if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {  in process_ident_missing_features()
2593 		con->error_msg = "protocol error, unexpected ident_missing_features";  in process_ident_missing_features()
2600 	con->error_msg = "missing required protocol features";  in process_ident_missing_features()
2608 static int process_session_reconnect_ok(struct ceph_connection *con,  in process_session_reconnect_ok()  argument
2613 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_reconnect_ok()
2614 		con->error_msg = "protocol error, unexpected session_reconnect_ok";  in process_session_reconnect_ok()
2620 	dout("%s con %p seq %llu\n", __func__, con, seq);  in process_session_reconnect_ok()
2621 	ceph_con_discard_requeued(con, seq);  in process_session_reconnect_ok()
2623 	clear_in_sign_kvecs(con);  in process_session_reconnect_ok()
2624 	clear_out_sign_kvecs(con);  in process_session_reconnect_ok()
2625 	free_conn_bufs(con);  in process_session_reconnect_ok()
2626 	con->delay = 0;  /* reset backoff memory */  in process_session_reconnect_ok()
2628 	con->state = CEPH_CON_S_OPEN;  in process_session_reconnect_ok()
2629 	con->v2.out_state = OUT_S_GET_NEXT;  in process_session_reconnect_ok()
2637 static int process_session_retry(struct ceph_connection *con,  in process_session_retry()  argument
2643 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_retry()
2644 		con->error_msg = "protocol error, unexpected session_retry";  in process_session_retry()
2650 	dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);  in process_session_retry()
2651 	WARN_ON(connect_seq <= con->v2.connect_seq);  in process_session_retry()
2652 	con->v2.connect_seq = connect_seq + 1;  in process_session_retry()
2654 	free_conn_bufs(con);  in process_session_retry()
2656 	reset_out_kvecs(con);  in process_session_retry()
2657 	ret = prepare_session_reconnect(con);  in process_session_retry()
2670 static int process_session_retry_global(struct ceph_connection *con,  in process_session_retry_global()  argument
2676 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_retry_global()
2677 		con->error_msg = "protocol error, unexpected session_retry_global";  in process_session_retry_global()
2683 	dout("%s con %p global_seq %llu\n", __func__, con, global_seq);  in process_session_retry_global()
2684 	WARN_ON(global_seq <= con->v2.global_seq);  in process_session_retry_global()
2685 	con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);  in process_session_retry_global()
2687 	free_conn_bufs(con);  in process_session_retry_global()
2689 	reset_out_kvecs(con);  in process_session_retry_global()
2690 	ret = prepare_session_reconnect(con);  in process_session_retry_global()
2703 static int process_session_reset(struct ceph_connection *con,  in process_session_reset()  argument
2709 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_reset()
2710 		con->error_msg = "protocol error, unexpected session_reset";  in process_session_reset()
2716 		con->error_msg = "protocol error, bad session_reset";  in process_session_reset()
2720 	pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),  in process_session_reset()
2721 		ceph_pr_addr(&con->peer_addr));  in process_session_reset()
2722 	ceph_con_reset_session(con);  in process_session_reset()
2724 	mutex_unlock(&con->mutex);  in process_session_reset()
2725 	if (con->ops->peer_reset)  in process_session_reset()
2726 		con->ops->peer_reset(con);  in process_session_reset()
2727 	mutex_lock(&con->mutex);  in process_session_reset()
2728 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_reset()
2729 		dout("%s con %p state changed to %d\n", __func__, con,  in process_session_reset()
2730 		     con->state);  in process_session_reset()
2734 	free_conn_bufs(con);  in process_session_reset()
2736 	reset_out_kvecs(con);  in process_session_reset()
2737 	ret = prepare_client_ident(con);  in process_session_reset()
2743 	con->state = CEPH_CON_S_V2_SESSION_CONNECT;  in process_session_reset()
2751 static int process_keepalive2_ack(struct ceph_connection *con,  in process_keepalive2_ack()  argument
2754 	if (con->state != CEPH_CON_S_OPEN) {  in process_keepalive2_ack()
2755 		con->error_msg = "protocol error, unexpected keepalive2_ack";  in process_keepalive2_ack()
2760 	ceph_decode_timespec64(&con->last_keepalive_ack, p);  in process_keepalive2_ack()
2762 	dout("%s con %p timestamp %lld.%09ld\n", __func__, con,  in process_keepalive2_ack()
2763 	     con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);  in process_keepalive2_ack()
2772 static int process_ack(struct ceph_connection *con, void *p, void *end)  in process_ack()  argument
2776 	if (con->state != CEPH_CON_S_OPEN) {  in process_ack()
2777 		con->error_msg = "protocol error, unexpected ack";  in process_ack()
2783 	dout("%s con %p seq %llu\n", __func__, con, seq);  in process_ack()
2784 	ceph_con_discard_sent(con, seq);  in process_ack()
2792 static int process_control(struct ceph_connection *con, void *p, void *end)  in process_control()  argument
2794 	int tag = con->v2.in_desc.fd_tag;  in process_control()
2797 	dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));  in process_control()
2801 		ret = process_hello(con, p, end);  in process_control()
2804 		ret = process_auth_bad_method(con, p, end);  in process_control()
2807 		ret = process_auth_reply_more(con, p, end);  in process_control()
2810 		ret = process_auth_done(con, p, end);  in process_control()
2813 		ret = process_auth_signature(con, p, end);  in process_control()
2816 		ret = process_server_ident(con, p, end);  in process_control()
2819 		ret = process_ident_missing_features(con, p, end);  in process_control()
2822 		ret = process_session_reconnect_ok(con, p, end);  in process_control()
2825 		ret = process_session_retry(con, p, end);  in process_control()
2828 		ret = process_session_retry_global(con, p, end);  in process_control()
2831 		ret = process_session_reset(con, p, end);  in process_control()
2834 		ret = process_keepalive2_ack(con, p, end);  in process_control()
2837 		ret = process_ack(con, p, end);  in process_control()
2841 		con->error_msg = "protocol error, bad tag";  in process_control()
2845 		dout("%s con %p error %d\n", __func__, con, ret);  in process_control()
2849 	prepare_read_preamble(con);  in process_control()
2855  *   1 - con->in_msg set, read message
2859 static int process_message_header(struct ceph_connection *con,  in process_message_header()  argument
2862 	struct ceph_frame_desc *desc = &con->v2.in_desc;  in process_message_header()
2871 	if ((s64)seq - (s64)con->in_seq < 1) {  in process_message_header()
2873 			ENTITY_NAME(con->peer_name),  in process_message_header()
2874 			ceph_pr_addr(&con->peer_addr),  in process_message_header()
2875 			seq, con->in_seq + 1);  in process_message_header()
2878 	if ((s64)seq - (s64)con->in_seq > 1) {  in process_message_header()
2879 		pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);  in process_message_header()
2880 		con->error_msg = "bad message sequence # for incoming message";  in process_message_header()
2884 	ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));  in process_message_header()
2887 		    desc->fd_lens[3], &con->peer_name);  in process_message_header()
2888 	ret = ceph_con_in_msg_alloc(con, &hdr, &skip);  in process_message_header()
2892 	WARN_ON(!con->in_msg ^ skip);  in process_message_header()
2896 	WARN_ON(!con->in_msg);  in process_message_header()
2897 	WARN_ON(con->in_msg->con != con);  in process_message_header()
2901 static int process_message(struct ceph_connection *con)  in process_message()  argument
2903 	ceph_con_process_message(con);  in process_message()
2907 	 * ceph_con_process_message() temporarily drops con->mutex.  in process_message()
2909 	if (con->state != CEPH_CON_S_OPEN) {  in process_message()
2910 		dout("%s con %p state changed to %d\n", __func__, con,  in process_message()
2911 		     con->state);  in process_message()
2915 	prepare_read_preamble(con);  in process_message()
2919 static int __handle_control(struct ceph_connection *con, void *p)  in __handle_control()  argument
2921 	void *end = p + con->v2.in_desc.fd_lens[0];  in __handle_control()
2925 	if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)  in __handle_control()
2926 		return process_control(con, p, end);  in __handle_control()
2928 	ret = process_message_header(con, p, end);  in __handle_control()
2932 		prepare_skip_message(con);  in __handle_control()
2936 	msg = con->in_msg;  /* set in process_message_header() */  in __handle_control()
2951 		return process_message(con);  in __handle_control()
2953 	if (con_secure(con))  in __handle_control()
2954 		return prepare_read_tail_secure(con);  in __handle_control()
2956 	return prepare_read_tail_plain(con);  in __handle_control()
2959 static int handle_preamble(struct ceph_connection *con)  in handle_preamble()  argument
2961 	struct ceph_frame_desc *desc = &con->v2.in_desc;  in handle_preamble()
2964 	if (con_secure(con)) {  in handle_preamble()
2965 		ret = decrypt_preamble(con);  in handle_preamble()
2968 				con->error_msg = "integrity error, bad preamble auth tag";  in handle_preamble()
2973 	ret = decode_preamble(con->v2.in_buf, desc);  in handle_preamble()
2976 			con->error_msg = "integrity error, bad crc";  in handle_preamble()
2978 			con->error_msg = "protocol error, bad preamble";  in handle_preamble()
2982 	dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,  in handle_preamble()
2983 	     con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],  in handle_preamble()
2986 	if (!con_secure(con))  in handle_preamble()
2987 		return prepare_read_control(con);  in handle_preamble()
2990 		return prepare_read_control_remainder(con);  in handle_preamble()
2992 	return __handle_control(con, CTRL_BODY(con->v2.in_buf));  in handle_preamble()
2995 static int handle_control(struct ceph_connection *con)  in handle_control()  argument
2997 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in handle_control()
3001 	WARN_ON(con_secure(con));  in handle_control()
3003 	ret = verify_control_crc(con);  in handle_control()
3005 		con->error_msg = "integrity error, bad crc";  in handle_control()
3009 	if (con->state == CEPH_CON_S_V2_AUTH) {  in handle_control()
3010 		buf = alloc_conn_buf(con, ctrl_len);  in handle_control()
3014 		memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);  in handle_control()
3015 		return __handle_control(con, buf);  in handle_control()
3018 	return __handle_control(con, con->v2.in_kvecs[0].iov_base);  in handle_control()
3021 static int handle_control_remainder(struct ceph_connection *con)  in handle_control_remainder()  argument
3025 	WARN_ON(!con_secure(con));  in handle_control_remainder()
3027 	ret = decrypt_control_remainder(con);  in handle_control_remainder()
3030 			con->error_msg = "integrity error, bad control remainder auth tag";  in handle_control_remainder()
3034 	return __handle_control(con, con->v2.in_kvecs[0].iov_base -  in handle_control_remainder()
3038 static int handle_epilogue(struct ceph_connection *con)  in handle_epilogue()  argument
3043 	if (con_secure(con)) {  in handle_epilogue()
3044 		ret = decrypt_tail(con);  in handle_epilogue()
3047 				con->error_msg = "integrity error, bad epilogue auth tag";  in handle_epilogue()
3052 		ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);  in handle_epilogue()
3054 			con->error_msg = "protocol error, bad epilogue";  in handle_epilogue()
3058 		ret = decode_epilogue(con->v2.in_buf, &front_crc,  in handle_epilogue()
3061 			con->error_msg = "protocol error, bad epilogue";  in handle_epilogue()
3065 		ret = verify_epilogue_crcs(con, front_crc, middle_crc,  in handle_epilogue()
3068 			con->error_msg = "integrity error, bad crc";  in handle_epilogue()
3073 	return process_message(con);  in handle_epilogue()
3076 static void finish_skip(struct ceph_connection *con)  in finish_skip()  argument
3078 	dout("%s con %p\n", __func__, con);  in finish_skip()
3080 	if (con_secure(con))  in finish_skip()
3081 		gcm_inc_nonce(&con->v2.in_gcm_nonce);  in finish_skip()
3083 	__finish_skip(con);  in finish_skip()
3086 static int populate_in_iter(struct ceph_connection *con)  in populate_in_iter()  argument
3090 	dout("%s con %p state %d in_state %d\n", __func__, con, con->state,  in populate_in_iter()
3091 	     con->v2.in_state);  in populate_in_iter()
3092 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in populate_in_iter()
3094 	if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {  in populate_in_iter()
3095 		ret = process_banner_prefix(con);  in populate_in_iter()
3096 	} else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {  in populate_in_iter()
3097 		ret = process_banner_payload(con);  in populate_in_iter()
3098 	} else if ((con->state >= CEPH_CON_S_V2_HELLO &&  in populate_in_iter()
3099 		    con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||  in populate_in_iter()
3100 		   con->state == CEPH_CON_S_OPEN) {  in populate_in_iter()
3101 		switch (con->v2.in_state) {  in populate_in_iter()
3103 			ret = handle_preamble(con);  in populate_in_iter()
3106 			ret = handle_control(con);  in populate_in_iter()
3109 			ret = handle_control_remainder(con);  in populate_in_iter()
3112 			ret = prepare_read_data(con);  in populate_in_iter()
3115 			prepare_read_data_cont(con);  in populate_in_iter()
3119 			prepare_read_enc_page(con);  in populate_in_iter()
3123 			ret = prepare_sparse_read_data(con);  in populate_in_iter()
3126 			ret = prepare_sparse_read_cont(con);  in populate_in_iter()
3129 			ret = handle_epilogue(con);  in populate_in_iter()
3132 			finish_skip(con);  in populate_in_iter()
3136 			WARN(1, "bad in_state %d", con->v2.in_state);  in populate_in_iter()
3140 		WARN(1, "bad state %d", con->state);  in populate_in_iter()
3144 		dout("%s con %p error %d\n", __func__, con, ret);  in populate_in_iter()
3148 	if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))  in populate_in_iter()
3150 	dout("%s con %p populated %zu\n", __func__, con,  in populate_in_iter()
3151 	     iov_iter_count(&con->v2.in_iter));  in populate_in_iter()
3155 int ceph_con_v2_try_read(struct ceph_connection *con)  in ceph_con_v2_try_read()  argument
3159 	dout("%s con %p state %d need %zu\n", __func__, con, con->state,  in ceph_con_v2_try_read()
3160 	     iov_iter_count(&con->v2.in_iter));  in ceph_con_v2_try_read()
3162 	if (con->state == CEPH_CON_S_PREOPEN)  in ceph_con_v2_try_read()
3170 	if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))  in ceph_con_v2_try_read()
3174 		ret = ceph_tcp_recv(con);  in ceph_con_v2_try_read()
3178 		ret = populate_in_iter(con);  in ceph_con_v2_try_read()
3180 			if (ret && ret != -EAGAIN && !con->error_msg)  in ceph_con_v2_try_read()
3181 				con->error_msg = "read processing error";  in ceph_con_v2_try_read()
3187 static void queue_data(struct ceph_connection *con)  in queue_data()  argument
3191 	con->v2.out_epil.data_crc = -1;  in queue_data()
3192 	ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,  in queue_data()
3193 				  data_len(con->out_msg));  in queue_data()
3195 	get_bvec_at(&con->v2.out_cursor, &bv);  in queue_data()
3196 	set_out_bvec(con, &bv, true);  in queue_data()
3197 	con->v2.out_state = OUT_S_QUEUE_DATA_CONT;  in queue_data()
3200 static void queue_data_cont(struct ceph_connection *con)  in queue_data_cont()  argument
3204 	con->v2.out_epil.data_crc = ceph_crc32c_page(  in queue_data_cont()
3205 		con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,  in queue_data_cont()
3206 		con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);  in queue_data_cont()
3208 	ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);  in queue_data_cont()
3209 	if (con->v2.out_cursor.total_resid) {  in queue_data_cont()
3210 		get_bvec_at(&con->v2.out_cursor, &bv);  in queue_data_cont()
3211 		set_out_bvec(con, &bv, true);  in queue_data_cont()
3212 		WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);  in queue_data_cont()
3220 	reset_out_kvecs(con);  in queue_data_cont()
3221 	prepare_epilogue_plain(con, false);  in queue_data_cont()
3222 	con->v2.out_state = OUT_S_FINISH_MESSAGE;  in queue_data_cont()
3225 static void queue_enc_page(struct ceph_connection *con)  in queue_enc_page()  argument
3229 	dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,  in queue_enc_page()
3230 	     con->v2.out_enc_resid);  in queue_enc_page()
3231 	WARN_ON(!con->v2.out_enc_resid);  in queue_enc_page()
3233 	bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i],  in queue_enc_page()
3234 		      min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0);  in queue_enc_page()
3236 	set_out_bvec(con, &bv, false);  in queue_enc_page()
3237 	con->v2.out_enc_i++;  in queue_enc_page()
3238 	con->v2.out_enc_resid -= bv.bv_len;  in queue_enc_page()
3240 	if (con->v2.out_enc_resid) {  in queue_enc_page()
3241 		WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);  in queue_enc_page()
3249 	WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);  in queue_enc_page()
3250 	con->v2.out_state = OUT_S_FINISH_MESSAGE;  in queue_enc_page()
3253 static void queue_zeros(struct ceph_connection *con)  in queue_zeros()  argument
3255 	dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);  in queue_zeros()
3257 	if (con->v2.out_zero) {  in queue_zeros()
3258 		set_out_bvec_zero(con);  in queue_zeros()
3259 		con->v2.out_zero -= con->v2.out_bvec.bv_len;  in queue_zeros()
3260 		con->v2.out_state = OUT_S_QUEUE_ZEROS;  in queue_zeros()
3269 	reset_out_kvecs(con);  in queue_zeros()
3270 	prepare_epilogue_plain(con, true);  in queue_zeros()
3271 	con->v2.out_state = OUT_S_FINISH_MESSAGE;  in queue_zeros()
3274 static void finish_message(struct ceph_connection *con)  in finish_message()  argument
3276 	dout("%s con %p msg %p\n", __func__, con, con->out_msg);  in finish_message()
3279 	if (con->v2.out_enc_pages) {  in finish_message()
3280 		WARN_ON(!con->v2.out_enc_page_cnt);  in finish_message()
3281 		ceph_release_page_vector(con->v2.out_enc_pages,  in finish_message()
3282 					 con->v2.out_enc_page_cnt);  in finish_message()
3283 		con->v2.out_enc_pages = NULL;  in finish_message()
3284 		con->v2.out_enc_page_cnt = 0;  in finish_message()
3287 	if (con->out_msg) {  in finish_message()
3288 		ceph_msg_put(con->out_msg);  in finish_message()
3289 		con->out_msg = NULL;  in finish_message()
3292 	con->v2.out_state = OUT_S_GET_NEXT;  in finish_message()
3295 static int populate_out_iter(struct ceph_connection *con)  in populate_out_iter()  argument
3299 	dout("%s con %p state %d out_state %d\n", __func__, con, con->state,  in populate_out_iter()
3300 	     con->v2.out_state);  in populate_out_iter()
3301 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in populate_out_iter()
3303 	if (con->state != CEPH_CON_S_OPEN) {  in populate_out_iter()
3304 		WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||  in populate_out_iter()
3305 			con->state > CEPH_CON_S_V2_SESSION_RECONNECT);  in populate_out_iter()
3309 	switch (con->v2.out_state) {  in populate_out_iter()
3311 		WARN_ON(!con->out_msg);  in populate_out_iter()
3312 		queue_data(con);  in populate_out_iter()
3315 		WARN_ON(!con->out_msg);  in populate_out_iter()
3316 		queue_data_cont(con);  in populate_out_iter()
3319 		queue_enc_page(con);  in populate_out_iter()
3322 		WARN_ON(con->out_msg);  /* revoked */  in populate_out_iter()
3323 		queue_zeros(con);  in populate_out_iter()
3326 		finish_message(con);  in populate_out_iter()
3331 		WARN(1, "bad out_state %d", con->v2.out_state);  in populate_out_iter()
3335 	WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);  in populate_out_iter()
3336 	if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {  in populate_out_iter()
3337 		ret = prepare_keepalive2(con);  in populate_out_iter()
3342 	} else if (!list_empty(&con->out_queue)) {  in populate_out_iter()
3343 		ceph_con_get_out_msg(con);  in populate_out_iter()
3344 		ret = prepare_message(con);  in populate_out_iter()
3349 	} else if (con->in_seq > con->in_seq_acked) {  in populate_out_iter()
3350 		ret = prepare_ack(con);  in populate_out_iter()
3360 	if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))  in populate_out_iter()
3362 	dout("%s con %p populated %zu\n", __func__, con,  in populate_out_iter()
3363 	     iov_iter_count(&con->v2.out_iter));  in populate_out_iter()
3367 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in populate_out_iter()
3368 	dout("%s con %p nothing pending\n", __func__, con);  in populate_out_iter()
3369 	ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);  in populate_out_iter()
3373 int ceph_con_v2_try_write(struct ceph_connection *con)  in ceph_con_v2_try_write()  argument
3377 	dout("%s con %p state %d have %zu\n", __func__, con, con->state,  in ceph_con_v2_try_write()
3378 	     iov_iter_count(&con->v2.out_iter));  in ceph_con_v2_try_write()
3381 	if (con->state == CEPH_CON_S_PREOPEN) {  in ceph_con_v2_try_write()
3382 		WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);  in ceph_con_v2_try_write()
3389 		con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);  in ceph_con_v2_try_write()
3390 		if (con->v2.server_cookie)  in ceph_con_v2_try_write()
3391 			con->v2.connect_seq++;  in ceph_con_v2_try_write()
3393 		ret = prepare_read_banner_prefix(con);  in ceph_con_v2_try_write()
3396 			con->error_msg = "connect error";  in ceph_con_v2_try_write()
3400 		reset_out_kvecs(con);  in ceph_con_v2_try_write()
3401 		ret = prepare_banner(con);  in ceph_con_v2_try_write()
3404 			con->error_msg = "connect error";  in ceph_con_v2_try_write()
3408 		ret = ceph_tcp_connect(con);  in ceph_con_v2_try_write()
3411 			con->error_msg = "connect error";  in ceph_con_v2_try_write()
3416 	if (!iov_iter_count(&con->v2.out_iter)) {  in ceph_con_v2_try_write()
3417 		ret = populate_out_iter(con);  in ceph_con_v2_try_write()
3419 			if (ret && ret != -EAGAIN && !con->error_msg)  in ceph_con_v2_try_write()
3420 				con->error_msg = "write processing error";  in ceph_con_v2_try_write()
3425 	tcp_sock_set_cork(con->sock->sk, true);  in ceph_con_v2_try_write()
3427 		ret = ceph_tcp_send(con);  in ceph_con_v2_try_write()
3431 		ret = populate_out_iter(con);  in ceph_con_v2_try_write()
3433 			if (ret && ret != -EAGAIN && !con->error_msg)  in ceph_con_v2_try_write()
3434 				con->error_msg = "write processing error";  in ceph_con_v2_try_write()
3439 	tcp_sock_set_cork(con->sock->sk, false);  in ceph_con_v2_try_write()
3456 static void prepare_zero_front(struct ceph_connection *con, int resid)  in prepare_zero_front()  argument
3460 	WARN_ON(!resid || resid > front_len(con->out_msg));  in prepare_zero_front()
3461 	sent = front_len(con->out_msg) - resid;  in prepare_zero_front()
3462 	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);  in prepare_zero_front()
3465 		con->v2.out_epil.front_crc =  in prepare_zero_front()
3466 			crc32c(-1, con->out_msg->front.iov_base, sent);  in prepare_zero_front()
3467 		con->v2.out_epil.front_crc =  in prepare_zero_front()
3468 			crc32c_zeros(con->v2.out_epil.front_crc, resid);  in prepare_zero_front()
3470 		con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);  in prepare_zero_front()
3473 	con->v2.out_iter.count -= resid;  in prepare_zero_front()
3474 	out_zero_add(con, resid);  in prepare_zero_front()
3477 static void prepare_zero_middle(struct ceph_connection *con, int resid)  in prepare_zero_middle()  argument
3481 	WARN_ON(!resid || resid > middle_len(con->out_msg));  in prepare_zero_middle()
3482 	sent = middle_len(con->out_msg) - resid;  in prepare_zero_middle()
3483 	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);  in prepare_zero_middle()
3486 		con->v2.out_epil.middle_crc =  in prepare_zero_middle()
3487 			crc32c(-1, con->out_msg->middle->vec.iov_base, sent);  in prepare_zero_middle()
3488 		con->v2.out_epil.middle_crc =  in prepare_zero_middle()
3489 			crc32c_zeros(con->v2.out_epil.middle_crc, resid);  in prepare_zero_middle()
3491 		con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);  in prepare_zero_middle()
3494 	con->v2.out_iter.count -= resid;  in prepare_zero_middle()
3495 	out_zero_add(con, resid);  in prepare_zero_middle()
3498 static void prepare_zero_data(struct ceph_connection *con)  in prepare_zero_data()  argument
3500 	dout("%s con %p\n", __func__, con);  in prepare_zero_data()
3501 	con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));  in prepare_zero_data()
3502 	out_zero_add(con, data_len(con->out_msg));  in prepare_zero_data()
3505 static void revoke_at_queue_data(struct ceph_connection *con)  in revoke_at_queue_data()  argument
3510 	WARN_ON(!data_len(con->out_msg));  in revoke_at_queue_data()
3511 	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));  in revoke_at_queue_data()
3512 	resid = iov_iter_count(&con->v2.out_iter);  in revoke_at_queue_data()
3514 	boundary = front_len(con->out_msg) + middle_len(con->out_msg);  in revoke_at_queue_data()
3518 		dout("%s con %p was sending head\n", __func__, con);  in revoke_at_queue_data()
3519 		if (front_len(con->out_msg))  in revoke_at_queue_data()
3520 			prepare_zero_front(con, front_len(con->out_msg));  in revoke_at_queue_data()
3521 		if (middle_len(con->out_msg))  in revoke_at_queue_data()
3522 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_queue_data()
3523 		prepare_zero_data(con);  in revoke_at_queue_data()
3524 		WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);  in revoke_at_queue_data()
3525 		con->v2.out_state = OUT_S_QUEUE_ZEROS;  in revoke_at_queue_data()
3529 	boundary = middle_len(con->out_msg);  in revoke_at_queue_data()
3532 		dout("%s con %p was sending front\n", __func__, con);  in revoke_at_queue_data()
3533 		prepare_zero_front(con, resid);  in revoke_at_queue_data()
3534 		if (middle_len(con->out_msg))  in revoke_at_queue_data()
3535 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_queue_data()
3536 		prepare_zero_data(con);  in revoke_at_queue_data()
3537 		queue_zeros(con);  in revoke_at_queue_data()
3542 	dout("%s con %p was sending middle\n", __func__, con);  in revoke_at_queue_data()
3543 	prepare_zero_middle(con, resid);  in revoke_at_queue_data()
3544 	prepare_zero_data(con);  in revoke_at_queue_data()
3545 	queue_zeros(con);  in revoke_at_queue_data()
3548 static void revoke_at_queue_data_cont(struct ceph_connection *con)  in revoke_at_queue_data_cont()  argument
3552 	WARN_ON(!data_len(con->out_msg));  in revoke_at_queue_data_cont()
3553 	WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));  in revoke_at_queue_data_cont()
3554 	resid = iov_iter_count(&con->v2.out_iter);  in revoke_at_queue_data_cont()
3555 	WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);  in revoke_at_queue_data_cont()
3556 	sent = con->v2.out_bvec.bv_len - resid;  in revoke_at_queue_data_cont()
3557 	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);  in revoke_at_queue_data_cont()
3560 		con->v2.out_epil.data_crc = ceph_crc32c_page(  in revoke_at_queue_data_cont()
3561 			con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,  in revoke_at_queue_data_cont()
3562 			con->v2.out_bvec.bv_offset, sent);  in revoke_at_queue_data_cont()
3563 		ceph_msg_data_advance(&con->v2.out_cursor, sent);  in revoke_at_queue_data_cont()
3565 	WARN_ON(resid > con->v2.out_cursor.total_resid);  in revoke_at_queue_data_cont()
3566 	con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,  in revoke_at_queue_data_cont()
3567 						con->v2.out_cursor.total_resid);  in revoke_at_queue_data_cont()
3569 	con->v2.out_iter.count -= resid;  in revoke_at_queue_data_cont()
3570 	out_zero_add(con, con->v2.out_cursor.total_resid);  in revoke_at_queue_data_cont()
3571 	queue_zeros(con);  in revoke_at_queue_data_cont()
3574 static void revoke_at_finish_message(struct ceph_connection *con)  in revoke_at_finish_message()  argument
3579 	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));  in revoke_at_finish_message()
3580 	resid = iov_iter_count(&con->v2.out_iter);  in revoke_at_finish_message()
3582 	if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&  in revoke_at_finish_message()
3583 	    !data_len(con->out_msg)) {  in revoke_at_finish_message()
3585 		dout("%s con %p was sending head (empty message) - noop\n",  in revoke_at_finish_message()
3586 		     __func__, con);  in revoke_at_finish_message()
3590 	boundary = front_len(con->out_msg) + middle_len(con->out_msg) +  in revoke_at_finish_message()
3595 		dout("%s con %p was sending head\n", __func__, con);  in revoke_at_finish_message()
3596 		if (front_len(con->out_msg))  in revoke_at_finish_message()
3597 			prepare_zero_front(con, front_len(con->out_msg));  in revoke_at_finish_message()
3598 		if (middle_len(con->out_msg))  in revoke_at_finish_message()
3599 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_finish_message()
3600 		con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3601 		WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);  in revoke_at_finish_message()
3602 		con->v2.out_state = OUT_S_QUEUE_ZEROS;  in revoke_at_finish_message()
3606 	boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3609 		dout("%s con %p was sending front\n", __func__, con);  in revoke_at_finish_message()
3610 		prepare_zero_front(con, resid);  in revoke_at_finish_message()
3611 		if (middle_len(con->out_msg))  in revoke_at_finish_message()
3612 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_finish_message()
3613 		con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3614 		queue_zeros(con);  in revoke_at_finish_message()
3621 		dout("%s con %p was sending middle\n", __func__, con);  in revoke_at_finish_message()
3622 		prepare_zero_middle(con, resid);  in revoke_at_finish_message()
3623 		con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3624 		queue_zeros(con);  in revoke_at_finish_message()
3629 	dout("%s con %p was sending epilogue - noop\n", __func__, con);  in revoke_at_finish_message()
3632 void ceph_con_v2_revoke(struct ceph_connection *con)  in ceph_con_v2_revoke()  argument
3634 	WARN_ON(con->v2.out_zero);  in ceph_con_v2_revoke()
3636 	if (con_secure(con)) {  in ceph_con_v2_revoke()
3637 		WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&  in ceph_con_v2_revoke()
3638 			con->v2.out_state != OUT_S_FINISH_MESSAGE);  in ceph_con_v2_revoke()
3639 		dout("%s con %p secure - noop\n", __func__, con);  in ceph_con_v2_revoke()
3643 	switch (con->v2.out_state) {  in ceph_con_v2_revoke()
3645 		revoke_at_queue_data(con);  in ceph_con_v2_revoke()
3648 		revoke_at_queue_data_cont(con);  in ceph_con_v2_revoke()
3651 		revoke_at_finish_message(con);  in ceph_con_v2_revoke()
3654 		WARN(1, "bad out_state %d", con->v2.out_state);  in ceph_con_v2_revoke()
3659 static void revoke_at_prepare_read_data(struct ceph_connection *con)  in revoke_at_prepare_read_data()  argument
3664 	WARN_ON(con_secure(con));  in revoke_at_prepare_read_data()
3665 	WARN_ON(!data_len(con->in_msg));  in revoke_at_prepare_read_data()
3666 	WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));  in revoke_at_prepare_read_data()
3667 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_read_data()
3670 	remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_prepare_read_data()
3671 	dout("%s con %p resid %d remaining %d\n", __func__, con, resid,  in revoke_at_prepare_read_data()
3673 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_read_data()
3674 	set_in_skip(con, resid + remaining);  in revoke_at_prepare_read_data()
3675 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_read_data()
3678 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)  in revoke_at_prepare_read_data_cont()  argument
3683 	WARN_ON(con_secure(con));  in revoke_at_prepare_read_data_cont()
3684 	WARN_ON(!data_len(con->in_msg));  in revoke_at_prepare_read_data_cont()
3685 	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));  in revoke_at_prepare_read_data_cont()
3686 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_read_data_cont()
3687 	WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);  in revoke_at_prepare_read_data_cont()
3688 	recved = con->v2.in_bvec.bv_len - resid;  in revoke_at_prepare_read_data_cont()
3689 	dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);  in revoke_at_prepare_read_data_cont()
3692 		ceph_msg_data_advance(&con->v2.in_cursor, recved);  in revoke_at_prepare_read_data_cont()
3693 	WARN_ON(resid > con->v2.in_cursor.total_resid);  in revoke_at_prepare_read_data_cont()
3696 	dout("%s con %p total_resid %zu remaining %d\n", __func__, con,  in revoke_at_prepare_read_data_cont()
3697 	     con->v2.in_cursor.total_resid, remaining);  in revoke_at_prepare_read_data_cont()
3698 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_read_data_cont()
3699 	set_in_skip(con, con->v2.in_cursor.total_resid + remaining);  in revoke_at_prepare_read_data_cont()
3700 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_read_data_cont()
3703 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)  in revoke_at_prepare_read_enc_page()  argument
3707 	WARN_ON(!con_secure(con));  in revoke_at_prepare_read_enc_page()
3708 	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));  in revoke_at_prepare_read_enc_page()
3709 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_read_enc_page()
3710 	WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);  in revoke_at_prepare_read_enc_page()
3712 	dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,  in revoke_at_prepare_read_enc_page()
3713 	     con->v2.in_enc_resid);  in revoke_at_prepare_read_enc_page()
3714 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_read_enc_page()
3715 	set_in_skip(con, resid + con->v2.in_enc_resid);  in revoke_at_prepare_read_enc_page()
3716 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_read_enc_page()
3719 static void revoke_at_prepare_sparse_data(struct ceph_connection *con)  in revoke_at_prepare_sparse_data()  argument
3724 	WARN_ON(con_secure(con));  in revoke_at_prepare_sparse_data()
3725 	WARN_ON(!data_len(con->in_msg));  in revoke_at_prepare_sparse_data()
3726 	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));  in revoke_at_prepare_sparse_data()
3727 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_sparse_data()
3728 	dout("%s con %p resid %d\n", __func__, con, resid);  in revoke_at_prepare_sparse_data()
3730 	remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain;  in revoke_at_prepare_sparse_data()
3731 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_sparse_data()
3732 	set_in_skip(con, resid + remaining);  in revoke_at_prepare_sparse_data()
3733 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_sparse_data()
3736 static void revoke_at_handle_epilogue(struct ceph_connection *con)  in revoke_at_handle_epilogue()  argument
3740 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_handle_epilogue()
3743 	dout("%s con %p resid %d\n", __func__, con, resid);  in revoke_at_handle_epilogue()
3744 	con->v2.in_iter.count -= resid;  in revoke_at_handle_epilogue()
3745 	set_in_skip(con, resid);  in revoke_at_handle_epilogue()
3746 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_handle_epilogue()
3749 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)  in ceph_con_v2_revoke_incoming()  argument
3751 	switch (con->v2.in_state) {  in ceph_con_v2_revoke_incoming()
3754 		revoke_at_prepare_read_data(con);  in ceph_con_v2_revoke_incoming()
3757 		revoke_at_prepare_read_data_cont(con);  in ceph_con_v2_revoke_incoming()
3760 		revoke_at_prepare_read_enc_page(con);  in ceph_con_v2_revoke_incoming()
3763 		revoke_at_prepare_sparse_data(con);  in ceph_con_v2_revoke_incoming()
3766 		revoke_at_handle_epilogue(con);  in ceph_con_v2_revoke_incoming()
3769 		WARN(1, "bad in_state %d", con->v2.in_state);  in ceph_con_v2_revoke_incoming()
3774 bool ceph_con_v2_opened(struct ceph_connection *con)  in ceph_con_v2_opened()  argument
3776 	return con->v2.peer_global_seq;  in ceph_con_v2_opened()
3779 void ceph_con_v2_reset_session(struct ceph_connection *con)  in ceph_con_v2_reset_session()  argument
3781 	con->v2.client_cookie = 0;  in ceph_con_v2_reset_session()
3782 	con->v2.server_cookie = 0;  in ceph_con_v2_reset_session()
3783 	con->v2.global_seq = 0;  in ceph_con_v2_reset_session()
3784 	con->v2.connect_seq = 0;  in ceph_con_v2_reset_session()
3785 	con->v2.peer_global_seq = 0;  in ceph_con_v2_reset_session()
3788 void ceph_con_v2_reset_protocol(struct ceph_connection *con)  in ceph_con_v2_reset_protocol()  argument
3790 	iov_iter_truncate(&con->v2.in_iter, 0);  in ceph_con_v2_reset_protocol()
3791 	iov_iter_truncate(&con->v2.out_iter, 0);  in ceph_con_v2_reset_protocol()
3792 	con->v2.out_zero = 0;  in ceph_con_v2_reset_protocol()
3794 	clear_in_sign_kvecs(con);  in ceph_con_v2_reset_protocol()
3795 	clear_out_sign_kvecs(con);  in ceph_con_v2_reset_protocol()
3796 	free_conn_bufs(con);  in ceph_con_v2_reset_protocol()
3798 	if (con->v2.in_enc_pages) {  in ceph_con_v2_reset_protocol()
3799 		WARN_ON(!con->v2.in_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3800 		ceph_release_page_vector(con->v2.in_enc_pages,  in ceph_con_v2_reset_protocol()
3801 					 con->v2.in_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3802 		con->v2.in_enc_pages = NULL;  in ceph_con_v2_reset_protocol()
3803 		con->v2.in_enc_page_cnt = 0;  in ceph_con_v2_reset_protocol()
3805 	if (con->v2.out_enc_pages) {  in ceph_con_v2_reset_protocol()
3806 		WARN_ON(!con->v2.out_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3807 		ceph_release_page_vector(con->v2.out_enc_pages,  in ceph_con_v2_reset_protocol()
3808 					 con->v2.out_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3809 		con->v2.out_enc_pages = NULL;  in ceph_con_v2_reset_protocol()
3810 		con->v2.out_enc_page_cnt = 0;  in ceph_con_v2_reset_protocol()
3813 	con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;  in ceph_con_v2_reset_protocol()
3814 	memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);  in ceph_con_v2_reset_protocol()
3815 	memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);  in ceph_con_v2_reset_protocol()
3817 	if (con->v2.hmac_tfm) {  in ceph_con_v2_reset_protocol()
3818 		crypto_free_shash(con->v2.hmac_tfm);  in ceph_con_v2_reset_protocol()
3819 		con->v2.hmac_tfm = NULL;  in ceph_con_v2_reset_protocol()
3821 	if (con->v2.gcm_req) {  in ceph_con_v2_reset_protocol()
3822 		aead_request_free(con->v2.gcm_req);  in ceph_con_v2_reset_protocol()
3823 		con->v2.gcm_req = NULL;  in ceph_con_v2_reset_protocol()
3825 	if (con->v2.gcm_tfm) {  in ceph_con_v2_reset_protocol()
3826 		crypto_free_aead(con->v2.gcm_tfm);  in ceph_con_v2_reset_protocol()
3827 		con->v2.gcm_tfm = NULL;  in ceph_con_v2_reset_protocol()