Lines Matching +full:dout +full:- +full:default
1 // SPDX-License-Identifier: GPL-2.0
89 if (ret == -EAGAIN) in do_recvmsg()
105 * 1 - done, nothing (else) to read
106 * 0 - socket is empty, need to wait
107 * <0 - error
113 dout("%s con %p %s %zu\n", __func__, con, in ceph_tcp_recv()
114 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need", in ceph_tcp_recv()
115 iov_iter_count(&con->v2.in_iter)); in ceph_tcp_recv()
116 ret = do_recvmsg(con->sock, &con->v2.in_iter); in ceph_tcp_recv()
117 dout("%s con %p ret %d left %zu\n", __func__, con, ret, in ceph_tcp_recv()
118 iov_iter_count(&con->v2.in_iter)); in ceph_tcp_recv()
131 if (ret == -EAGAIN) in do_sendmsg()
150 return -EINVAL; in do_try_sendpage()
154 bvec_set_page(&bv, it->bvec->bv_page, in do_try_sendpage()
156 it->bvec->bv_len - it->iov_offset), in do_try_sendpage()
157 it->bvec->bv_offset + it->iov_offset); in do_try_sendpage()
176 if (ret == -EAGAIN) in do_try_sendpage()
192 * 1 - done, nothing (else) to write
193 * 0 - socket is full, need to wait
194 * <0 - error
200 dout("%s con %p have %zu try_sendpage %d\n", __func__, con, in ceph_tcp_send()
201 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage); in ceph_tcp_send()
202 if (con->v2.out_iter_sendpage) in ceph_tcp_send()
203 ret = do_try_sendpage(con->sock, &con->v2.out_iter); in ceph_tcp_send()
205 ret = do_sendmsg(con->sock, &con->v2.out_iter); in ceph_tcp_send()
206 dout("%s con %p ret %d left %zu\n", __func__, con, ret, in ceph_tcp_send()
207 iov_iter_count(&con->v2.out_iter)); in ceph_tcp_send()
213 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs)); in add_in_kvec()
214 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); in add_in_kvec()
216 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf; in add_in_kvec()
217 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len; in add_in_kvec()
218 con->v2.in_kvec_cnt++; in add_in_kvec()
220 con->v2.in_iter.nr_segs++; in add_in_kvec()
221 con->v2.in_iter.count += len; in add_in_kvec()
226 WARN_ON(iov_iter_count(&con->v2.in_iter)); in reset_in_kvecs()
228 con->v2.in_kvec_cnt = 0; in reset_in_kvecs()
229 iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0); in reset_in_kvecs()
234 WARN_ON(iov_iter_count(&con->v2.in_iter)); in set_in_bvec()
236 con->v2.in_bvec = *bv; in set_in_bvec()
237 iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len); in set_in_bvec()
242 WARN_ON(iov_iter_count(&con->v2.in_iter)); in set_in_skip()
244 dout("%s con %p len %d\n", __func__, con, len); in set_in_skip()
245 iov_iter_discard(&con->v2.in_iter, ITER_DEST, len); in set_in_skip()
250 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs)); in add_out_kvec()
251 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); in add_out_kvec()
252 WARN_ON(con->v2.out_zero); in add_out_kvec()
254 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf; in add_out_kvec()
255 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len; in add_out_kvec()
256 con->v2.out_kvec_cnt++; in add_out_kvec()
258 con->v2.out_iter.nr_segs++; in add_out_kvec()
259 con->v2.out_iter.count += len; in add_out_kvec()
264 WARN_ON(iov_iter_count(&con->v2.out_iter)); in reset_out_kvecs()
265 WARN_ON(con->v2.out_zero); in reset_out_kvecs()
267 con->v2.out_kvec_cnt = 0; in reset_out_kvecs()
269 iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0); in reset_out_kvecs()
270 con->v2.out_iter_sendpage = false; in reset_out_kvecs()
276 WARN_ON(iov_iter_count(&con->v2.out_iter)); in set_out_bvec()
277 WARN_ON(con->v2.out_zero); in set_out_bvec()
279 con->v2.out_bvec = *bv; in set_out_bvec()
280 con->v2.out_iter_sendpage = zerocopy; in set_out_bvec()
281 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1, in set_out_bvec()
282 con->v2.out_bvec.bv_len); in set_out_bvec()
287 WARN_ON(iov_iter_count(&con->v2.out_iter)); in set_out_bvec_zero()
288 WARN_ON(!con->v2.out_zero); in set_out_bvec_zero()
290 bvec_set_page(&con->v2.out_bvec, ceph_zero_page, in set_out_bvec_zero()
291 min(con->v2.out_zero, (int)PAGE_SIZE), 0); in set_out_bvec_zero()
292 con->v2.out_iter_sendpage = true; in set_out_bvec_zero()
293 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1, in set_out_bvec_zero()
294 con->v2.out_bvec.bv_len); in set_out_bvec_zero()
299 dout("%s con %p len %d\n", __func__, con, len); in out_zero_add()
300 con->v2.out_zero += len; in out_zero_add()
307 dout("%s con %p len %d\n", __func__, con, len); in alloc_conn_buf()
309 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs))) in alloc_conn_buf()
316 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf; in alloc_conn_buf()
322 while (con->v2.conn_buf_cnt) in free_conn_bufs()
323 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]); in free_conn_bufs()
328 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs)); in add_in_sign_kvec()
330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf; in add_in_sign_kvec()
331 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len; in add_in_sign_kvec()
332 con->v2.in_sign_kvec_cnt++; in add_in_sign_kvec()
337 con->v2.in_sign_kvec_cnt = 0; in clear_in_sign_kvecs()
342 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs)); in add_out_sign_kvec()
344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf; in add_out_sign_kvec()
345 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len; in add_out_sign_kvec()
346 con->v2.out_sign_kvec_cnt++; in add_out_sign_kvec()
351 con->v2.out_sign_kvec_cnt = 0; in clear_out_sign_kvecs()
356 return con->v2.con_mode == CEPH_CON_MODE_SECURE; in con_secure()
361 return le32_to_cpu(msg->hdr.front_len); in front_len()
366 return le32_to_cpu(msg->hdr.middle_len); in middle_len()
371 return le32_to_cpu(msg->hdr.data_len); in data_len()
386 return padded_len(len) - len; in padding_len()
400 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; in head_onwire_len()
456 for (i = len_cnt - 1; i >= 0; i--) { in calc_segment_count()
471 desc->fd_tag = tag; in init_frame_desc()
472 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt); in init_frame_desc()
473 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT); in init_frame_desc()
474 for (i = 0; i < desc->fd_seg_cnt; i++) { in init_frame_desc()
475 desc->fd_lens[i] = lens[i]; in init_frame_desc()
476 desc->fd_aligns[i] = frame_aligns[i]; in init_frame_desc()
487 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; in encode_preamble()
493 ceph_encode_8(&p, desc->fd_tag); in encode_preamble()
494 ceph_encode_8(&p, desc->fd_seg_cnt); in encode_preamble()
495 for (i = 0; i < desc->fd_seg_cnt; i++) { in encode_preamble()
496 ceph_encode_32(&p, desc->fd_lens[i]); in encode_preamble()
497 ceph_encode_16(&p, desc->fd_aligns[i]); in encode_preamble()
500 put_unaligned_le32(crc32c(0, start, crcp - start), crcp); in encode_preamble()
505 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; in decode_preamble()
509 crc = crc32c(0, p, crcp - p); in decode_preamble()
514 return -EBADMSG; in decode_preamble()
519 desc->fd_tag = ceph_decode_8(&p); in decode_preamble()
520 desc->fd_seg_cnt = ceph_decode_8(&p); in decode_preamble()
521 if (desc->fd_seg_cnt < 1 || in decode_preamble()
522 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) { in decode_preamble()
523 pr_err("bad segment count %d\n", desc->fd_seg_cnt); in decode_preamble()
524 return -EINVAL; in decode_preamble()
526 for (i = 0; i < desc->fd_seg_cnt; i++) { in decode_preamble()
527 desc->fd_lens[i] = ceph_decode_32(&p); in decode_preamble()
528 desc->fd_aligns[i] = ceph_decode_16(&p); in decode_preamble()
531 if (desc->fd_lens[0] < 0 || in decode_preamble()
532 desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) { in decode_preamble()
533 pr_err("bad control segment length %d\n", desc->fd_lens[0]); in decode_preamble()
534 return -EINVAL; in decode_preamble()
536 if (desc->fd_lens[1] < 0 || in decode_preamble()
537 desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) { in decode_preamble()
538 pr_err("bad front segment length %d\n", desc->fd_lens[1]); in decode_preamble()
539 return -EINVAL; in decode_preamble()
541 if (desc->fd_lens[2] < 0 || in decode_preamble()
542 desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) { in decode_preamble()
543 pr_err("bad middle segment length %d\n", desc->fd_lens[2]); in decode_preamble()
544 return -EINVAL; in decode_preamble()
546 if (desc->fd_lens[3] < 0 || in decode_preamble()
547 desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) { in decode_preamble()
548 pr_err("bad data segment length %d\n", desc->fd_lens[3]); in decode_preamble()
549 return -EINVAL; in decode_preamble()
556 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) { in decode_preamble()
558 desc->fd_seg_cnt); in decode_preamble()
559 return -EINVAL; in decode_preamble()
567 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : in encode_epilogue_plain()
569 cpu_to_le32s(&con->v2.out_epil.front_crc); in encode_epilogue_plain()
570 cpu_to_le32s(&con->v2.out_epil.middle_crc); in encode_epilogue_plain()
571 cpu_to_le32s(&con->v2.out_epil.data_crc); in encode_epilogue_plain()
576 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil)); in encode_epilogue_secure()
577 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : in encode_epilogue_secure()
591 return -EINVAL; in decode_epilogue()
608 hdr->seq = hdr2->seq; in fill_header()
609 hdr->tid = hdr2->tid; in fill_header()
610 hdr->type = hdr2->type; in fill_header()
611 hdr->priority = hdr2->priority; in fill_header()
612 hdr->version = hdr2->version; in fill_header()
613 hdr->front_len = cpu_to_le32(front_len); in fill_header()
614 hdr->middle_len = cpu_to_le32(middle_len); in fill_header()
615 hdr->data_len = cpu_to_le32(data_len); in fill_header()
616 hdr->data_off = hdr2->data_off; in fill_header()
617 hdr->src = *peer_name; in fill_header()
618 hdr->compat_version = hdr2->compat_version; in fill_header()
619 hdr->reserved = 0; in fill_header()
620 hdr->crc = 0; in fill_header()
626 hdr2->seq = hdr->seq; in fill_header2()
627 hdr2->tid = hdr->tid; in fill_header2()
628 hdr2->type = hdr->type; in fill_header2()
629 hdr2->priority = hdr->priority; in fill_header2()
630 hdr2->version = hdr->version; in fill_header2()
631 hdr2->data_pre_padding_len = 0; in fill_header2()
632 hdr2->data_off = hdr->data_off; in fill_header2()
633 hdr2->ack_seq = cpu_to_le64(ack_seq); in fill_header2()
634 hdr2->flags = 0; in fill_header2()
635 hdr2->compat_version = hdr->compat_version; in fill_header2()
636 hdr2->reserved = 0; in fill_header2()
641 int ctrl_len = con->v2.in_desc.fd_lens[0]; in verify_control_crc()
644 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len); in verify_control_crc()
645 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN); in verify_control_crc()
647 crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len); in verify_control_crc()
648 expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base); in verify_control_crc()
652 return -EBADMSG; in verify_control_crc()
661 if (front_len(con->in_msg)) { in verify_epilogue_crcs()
662 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base, in verify_epilogue_crcs()
663 front_len(con->in_msg)); in verify_epilogue_crcs()
665 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg)); in verify_epilogue_crcs()
666 con->in_front_crc = -1; in verify_epilogue_crcs()
669 if (middle_len(con->in_msg)) in verify_epilogue_crcs()
670 con->in_middle_crc = crc32c(-1, in verify_epilogue_crcs()
671 con->in_msg->middle->vec.iov_base, in verify_epilogue_crcs()
672 middle_len(con->in_msg)); in verify_epilogue_crcs()
673 else if (data_len(con->in_msg)) in verify_epilogue_crcs()
674 con->in_middle_crc = -1; in verify_epilogue_crcs()
676 con->in_middle_crc = 0; in verify_epilogue_crcs()
678 if (!data_len(con->in_msg)) in verify_epilogue_crcs()
679 con->in_data_crc = 0; in verify_epilogue_crcs()
681 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg, in verify_epilogue_crcs()
682 con->in_front_crc, con->in_middle_crc, con->in_data_crc); in verify_epilogue_crcs()
684 if (con->in_front_crc != front_crc) { in verify_epilogue_crcs()
686 con->in_front_crc, front_crc); in verify_epilogue_crcs()
687 return -EBADMSG; in verify_epilogue_crcs()
689 if (con->in_middle_crc != middle_crc) { in verify_epilogue_crcs()
691 con->in_middle_crc, middle_crc); in verify_epilogue_crcs()
692 return -EBADMSG; in verify_epilogue_crcs()
694 if (con->in_data_crc != data_crc) { in verify_epilogue_crcs()
696 con->in_data_crc, data_crc); in verify_epilogue_crcs()
697 return -EBADMSG; in verify_epilogue_crcs()
710 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n", in setup_crypto()
711 __func__, con, con->v2.con_mode, session_key_len, con_secret_len); in setup_crypto()
712 WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req); in setup_crypto()
714 if (con->v2.con_mode != CEPH_CON_MODE_CRC && in setup_crypto()
715 con->v2.con_mode != CEPH_CON_MODE_SECURE) { in setup_crypto()
716 pr_err("bad con_mode %d\n", con->v2.con_mode); in setup_crypto()
717 return -EINVAL; in setup_crypto()
721 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC); in setup_crypto()
727 con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); in setup_crypto()
729 if (IS_ERR(con->v2.hmac_tfm)) { in setup_crypto()
730 ret = PTR_ERR(con->v2.hmac_tfm); in setup_crypto()
731 con->v2.hmac_tfm = NULL; in setup_crypto()
736 ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key, in setup_crypto()
743 if (con->v2.con_mode == CEPH_CON_MODE_CRC) { in setup_crypto()
750 return -EINVAL; in setup_crypto()
754 con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0); in setup_crypto()
756 if (IS_ERR(con->v2.gcm_tfm)) { in setup_crypto()
757 ret = PTR_ERR(con->v2.gcm_tfm); in setup_crypto()
758 con->v2.gcm_tfm = NULL; in setup_crypto()
764 crypto_aead_alignmask(con->v2.gcm_tfm)); in setup_crypto()
765 ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN); in setup_crypto()
771 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN); in setup_crypto()
772 ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN); in setup_crypto()
778 con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO); in setup_crypto()
779 if (!con->v2.gcm_req) { in setup_crypto()
781 return -ENOMEM; in setup_crypto()
784 crypto_init_wait(&con->v2.gcm_wait); in setup_crypto()
785 aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG, in setup_crypto()
786 crypto_req_done, &con->v2.gcm_wait); in setup_crypto()
788 memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN, in setup_crypto()
790 memcpy(&con->v2.out_gcm_nonce, in setup_crypto()
799 SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */ in hmac_sha256()
803 dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con, in hmac_sha256()
804 con->v2.hmac_tfm, kvec_cnt); in hmac_sha256()
806 if (!con->v2.hmac_tfm) { in hmac_sha256()
811 desc->tfm = con->v2.hmac_tfm; in hmac_sha256()
834 counter = le64_to_cpu(nonce->counter); in gcm_inc_nonce()
835 nonce->counter = cpu_to_le64(counter + 1); in gcm_inc_nonce()
845 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce; in gcm_crypt()
847 aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */ in gcm_crypt()
848 aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce); in gcm_crypt()
849 ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) : in gcm_crypt()
850 crypto_aead_decrypt(con->v2.gcm_req), in gcm_crypt()
851 &con->v2.gcm_wait); in gcm_crypt()
865 WARN_ON(!cursor->total_resid); in get_bvec_at()
867 /* skip zero-length data items */ in get_bvec_at()
868 while (!cursor->resid) in get_bvec_at()
896 int data_len = cursor->total_resid; in calc_sg_cnt_cursor()
909 } while (cursor->total_resid); in calc_sg_cnt_cursor()
928 len = min_t(int, end - p, PAGE_SIZE); in init_sgs()
948 int data_len = cursor->total_resid; in init_sgs_cursor()
960 } while (cursor->total_resid); in init_sgs_cursor()
987 int len = min(resid, (int)PAGE_SIZE - off); in init_sgs_pages()
993 resid -= len; in init_sgs_pages()
1018 sg_cnt += calc_sg_cnt(msg->front.iov_base, in setup_message_sgs()
1021 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base, in setup_message_sgs()
1038 cur_sg = sgt->sgl; in setup_message_sgs()
1040 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg), in setup_message_sgs()
1043 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg), in setup_message_sgs()
1064 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN); in decrypt_preamble()
1070 int ctrl_len = con->v2.in_desc.fd_lens[0]; in decrypt_control_remainder()
1071 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; in decrypt_control_remainder()
1075 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len); in decrypt_control_remainder()
1076 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len); in decrypt_control_remainder()
1079 sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len); in decrypt_control_remainder()
1080 sg_set_buf(&sgs[1], con->v2.in_buf, pt_len); in decrypt_control_remainder()
1090 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor; in process_v2_sparse_read()
1096 ret = con->ops->sparse_read(con, cursor, &buf); in process_v2_sparse_read()
1100 dout("%s: sparse_read return %x buf %p\n", __func__, ret, buf); in process_v2_sparse_read()
1105 struct page *spage = con->v2.in_enc_pages[idx]; in process_v2_sparse_read()
1106 int len = min_t(int, ret, PAGE_SIZE - soff); in process_v2_sparse_read()
1121 ret -= len; in process_v2_sparse_read()
1131 bool sparse = !!con->in_msg->sparse_read_total; in decrypt_tail()
1136 tail_len = tail_onwire_len(con->in_msg, true); in decrypt_tail()
1137 ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages, in decrypt_tail()
1138 con->v2.in_enc_page_cnt, 0, tail_len, in decrypt_tail()
1144 dpos = padded_len(front_len(con->in_msg) + padded_len(middle_len(con->in_msg))); in decrypt_tail()
1145 pages = con->v2.in_enc_pages; in decrypt_tail()
1148 ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf), in decrypt_tail()
1149 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf), in decrypt_tail()
1150 con->v2.in_buf, pages, dpos, true); in decrypt_tail()
1154 dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con, in decrypt_tail()
1155 con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents); in decrypt_tail()
1160 if (sparse && data_len(con->in_msg)) { in decrypt_tail()
1161 ret = process_v2_sparse_read(con, con->v2.in_enc_pages, dpos); in decrypt_tail()
1166 WARN_ON(!con->v2.in_enc_page_cnt); in decrypt_tail()
1167 ceph_release_page_vector(con->v2.in_enc_pages, in decrypt_tail()
1168 con->v2.in_enc_page_cnt); in decrypt_tail()
1169 con->v2.in_enc_pages = NULL; in decrypt_tail()
1170 con->v2.in_enc_page_cnt = 0; in decrypt_tail()
1185 return -ENOMEM; in prepare_banner()
1222 void *crcp = base + base_len - CEPH_CRC_LEN; in prepare_head_plain()
1225 crc = crc32c(-1, CTRL_BODY(base), ctrl_len); in prepare_head_plain()
1237 add_out_kvec(con, base, crcp - base); in prepare_head_plain()
1241 add_out_sign_kvec(con, base, crcp - base); in prepare_head_plain()
1256 CEPH_PREAMBLE_INLINE_LEN - ctrl_len); in prepare_head_secure_small()
1260 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN); in prepare_head_secure_small()
1282 * control body (ctrl_len - 48 bytes)
1291 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; in prepare_head_secure_big()
1299 sg_set_buf(&sgs[0], base, rem - base); in prepare_head_secure_big()
1301 ret = gcm_crypt(con, true, sgs, sgs, rem - base); in prepare_head_secure_big()
1309 sg_init_one(&sgs[0], rem, pmbl_tag - rem); in prepare_head_secure_big()
1310 ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem); in prepare_head_secure_big()
1314 add_out_kvec(con, base, rem - base); in prepare_head_secure_big()
1316 add_out_kvec(con, rem, pmbl_tag - rem); in prepare_head_secure_big()
1328 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag, in __prepare_control()
1333 return -EINVAL; in __prepare_control()
1340 return -EINVAL; in __prepare_control()
1370 ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr); in prepare_hello()
1373 return -ENOMEM; in prepare_hello()
1377 ceph_encode_entity_addr(&p, &con->peer_addr); in prepare_hello()
1385 #define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1397 return -ENOMEM; in prepare_auth_request()
1399 mutex_unlock(&con->mutex); in prepare_auth_request()
1400 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len, in prepare_auth_request()
1402 mutex_lock(&con->mutex); in prepare_auth_request()
1403 if (con->state != CEPH_CON_S_V2_HELLO) { in prepare_auth_request()
1404 dout("%s con %p state changed to %d\n", __func__, con, in prepare_auth_request()
1405 con->state); in prepare_auth_request()
1406 return -EAGAIN; in prepare_auth_request()
1409 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret); in prepare_auth_request()
1415 return -ENOMEM; in prepare_auth_request()
1434 return -ENOMEM; in prepare_auth_request_more()
1436 mutex_unlock(&con->mutex); in prepare_auth_request_more()
1437 ret = con->ops->handle_auth_reply_more(con, reply, reply_len, in prepare_auth_request_more()
1440 mutex_lock(&con->mutex); in prepare_auth_request_more()
1441 if (con->state != CEPH_CON_S_V2_AUTH) { in prepare_auth_request_more()
1442 dout("%s con %p state changed to %d\n", __func__, con, in prepare_auth_request_more()
1443 con->state); in prepare_auth_request_more()
1444 return -EAGAIN; in prepare_auth_request_more()
1447 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret); in prepare_auth_request_more()
1463 return -ENOMEM; in prepare_auth_signature()
1465 ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt, in prepare_auth_signature()
1476 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; in prepare_client_ident()
1477 struct ceph_client *client = from_msgr(con->msgr); in prepare_client_ident()
1482 WARN_ON(con->v2.server_cookie); in prepare_client_ident()
1483 WARN_ON(con->v2.connect_seq); in prepare_client_ident()
1484 WARN_ON(con->v2.peer_global_seq); in prepare_client_ident()
1486 if (!con->v2.client_cookie) { in prepare_client_ident()
1488 get_random_bytes(&con->v2.client_cookie, in prepare_client_ident()
1489 sizeof(con->v2.client_cookie)); in prepare_client_ident()
1490 } while (!con->v2.client_cookie); in prepare_client_ident()
1491 dout("%s con %p generated cookie 0x%llx\n", __func__, con, in prepare_client_ident()
1492 con->v2.client_cookie); in prepare_client_ident()
1494 dout("%s con %p cookie already set 0x%llx\n", __func__, con, in prepare_client_ident()
1495 con->v2.client_cookie); in prepare_client_ident()
1498 …dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx requi… in prepare_client_ident()
1499 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), in prepare_client_ident()
1500 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce), in prepare_client_ident()
1501 global_id, con->v2.global_seq, client->supported_features, in prepare_client_ident()
1502 client->required_features, con->v2.client_cookie); in prepare_client_ident()
1505 ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8; in prepare_client_ident()
1508 return -ENOMEM; in prepare_client_ident()
1514 ceph_encode_entity_addr(&p, &con->peer_addr); in prepare_client_ident()
1516 ceph_encode_64(&p, con->v2.global_seq); in prepare_client_ident()
1517 ceph_encode_64(&p, client->supported_features); in prepare_client_ident()
1518 ceph_encode_64(&p, client->required_features); in prepare_client_ident()
1520 ceph_encode_64(&p, con->v2.client_cookie); in prepare_client_ident()
1528 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; in prepare_session_reconnect()
1532 WARN_ON(!con->v2.client_cookie); in prepare_session_reconnect()
1533 WARN_ON(!con->v2.server_cookie); in prepare_session_reconnect()
1534 WARN_ON(!con->v2.connect_seq); in prepare_session_reconnect()
1535 WARN_ON(!con->v2.peer_global_seq); in prepare_session_reconnect()
1537 …dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_se… in prepare_session_reconnect()
1538 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), in prepare_session_reconnect()
1539 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq, in prepare_session_reconnect()
1540 con->v2.connect_seq, con->in_seq); in prepare_session_reconnect()
1545 return -ENOMEM; in prepare_session_reconnect()
1551 ceph_encode_64(&p, con->v2.client_cookie); in prepare_session_reconnect()
1552 ceph_encode_64(&p, con->v2.server_cookie); in prepare_session_reconnect()
1553 ceph_encode_64(&p, con->v2.global_seq); in prepare_session_reconnect()
1554 ceph_encode_64(&p, con->v2.connect_seq); in prepare_session_reconnect()
1555 ceph_encode_64(&p, con->in_seq); in prepare_session_reconnect()
1563 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf); in prepare_keepalive2()
1567 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec, in prepare_keepalive2()
1573 return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf, in prepare_keepalive2()
1581 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, in prepare_ack()
1582 con->in_seq_acked, con->in_seq); in prepare_ack()
1583 con->in_seq_acked = con->in_seq; in prepare_ack()
1585 p = CTRL_BODY(con->v2.out_buf); in prepare_ack()
1586 ceph_encode_64(&p, con->in_seq_acked); in prepare_ack()
1589 return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8); in prepare_ack()
1594 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con, in prepare_epilogue_plain()
1595 con->out_msg, aborted, con->v2.out_epil.front_crc, in prepare_epilogue_plain()
1596 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc); in prepare_epilogue_plain()
1599 add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN); in prepare_epilogue_plain()
1603 * For "used" empty segments, crc is -1. For unused (trailing)
1608 struct ceph_msg *msg = con->out_msg; in prepare_message_plain()
1610 prepare_head_plain(con, con->v2.out_buf, in prepare_message_plain()
1617 * we are done -- there is no epilogue. in prepare_message_plain()
1619 con->v2.out_state = OUT_S_FINISH_MESSAGE; in prepare_message_plain()
1623 con->v2.out_epil.front_crc = -1; in prepare_message_plain()
1624 con->v2.out_epil.middle_crc = -1; in prepare_message_plain()
1625 con->v2.out_state = OUT_S_QUEUE_DATA; in prepare_message_plain()
1630 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base, in prepare_message_plain()
1632 add_out_kvec(con, msg->front.iov_base, front_len(msg)); in prepare_message_plain()
1635 con->v2.out_epil.front_crc = -1; in prepare_message_plain()
1639 con->v2.out_epil.middle_crc = in prepare_message_plain()
1640 crc32c(-1, msg->middle->vec.iov_base, middle_len(msg)); in prepare_message_plain()
1641 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); in prepare_message_plain()
1643 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0; in prepare_message_plain()
1647 con->v2.out_state = OUT_S_QUEUE_DATA; in prepare_message_plain()
1649 con->v2.out_epil.data_crc = 0; in prepare_message_plain()
1651 con->v2.out_state = OUT_S_FINISH_MESSAGE; in prepare_message_plain()
1672 ret = prepare_head_secure_small(con, con->v2.out_buf, in prepare_message_secure()
1677 tail_len = tail_onwire_len(con->out_msg, true); in prepare_message_secure()
1681 * we are done -- there is no epilogue. in prepare_message_secure()
1683 con->v2.out_state = OUT_S_FINISH_MESSAGE; in prepare_message_secure()
1688 ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop, in prepare_message_secure()
1689 &con->v2.out_epil, NULL, 0, false); in prepare_message_secure()
1700 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt); in prepare_message_secure()
1701 con->v2.out_enc_pages = enc_pages; in prepare_message_secure()
1702 con->v2.out_enc_page_cnt = enc_page_cnt; in prepare_message_secure()
1703 con->v2.out_enc_resid = tail_len; in prepare_message_secure()
1704 con->v2.out_enc_i = 0; in prepare_message_secure()
1712 tail_len - CEPH_GCM_TAG_LEN); in prepare_message_secure()
1716 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con, in prepare_message_secure()
1717 con->out_msg, sgt.orig_nents, enc_page_cnt); in prepare_message_secure()
1718 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE; in prepare_message_secure()
1730 front_len(con->out_msg), in prepare_message()
1731 middle_len(con->out_msg), in prepare_message()
1732 data_len(con->out_msg) in prepare_message()
1737 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con, in prepare_message()
1738 con->out_msg, lens[0], lens[1], lens[2], lens[3]); in prepare_message()
1740 if (con->in_seq > con->in_seq_acked) { in prepare_message()
1741 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, in prepare_message()
1742 con->in_seq_acked, con->in_seq); in prepare_message()
1743 con->in_seq_acked = con->in_seq; in prepare_message()
1748 encode_preamble(&desc, con->v2.out_buf); in prepare_message()
1749 fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr, in prepare_message()
1750 con->in_seq_acked); in prepare_message()
1770 return -ENOMEM; in prepare_read_banner_prefix()
1775 con->state = CEPH_CON_S_V2_BANNER_PREFIX; in prepare_read_banner_prefix()
1786 return -ENOMEM; in prepare_read_banner_payload()
1791 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD; in prepare_read_banner_payload()
1798 add_in_kvec(con, con->v2.in_buf, in prepare_read_preamble()
1801 con->v2.in_state = IN_S_HANDLE_PREAMBLE; in prepare_read_preamble()
1806 int ctrl_len = con->v2.in_desc.fd_lens[0]; in prepare_read_control()
1811 if (con->state == CEPH_CON_S_V2_HELLO || in prepare_read_control()
1812 con->state == CEPH_CON_S_V2_AUTH) { in prepare_read_control()
1816 return -ENOMEM; in prepare_read_control()
1819 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN); in prepare_read_control()
1828 return -ENOMEM; in prepare_read_control()
1832 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len); in prepare_read_control()
1834 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN); in prepare_read_control()
1836 con->v2.in_state = IN_S_HANDLE_CONTROL; in prepare_read_control()
1842 int ctrl_len = con->v2.in_desc.fd_lens[0]; in prepare_read_control_remainder()
1843 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; in prepare_read_control_remainder()
1848 return -ENOMEM; in prepare_read_control_remainder()
1850 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN); in prepare_read_control_remainder()
1854 add_in_kvec(con, con->v2.in_buf, in prepare_read_control_remainder()
1856 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER; in prepare_read_control_remainder()
1864 con->in_data_crc = -1; in prepare_read_data()
1865 ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg, in prepare_read_data()
1866 data_len(con->in_msg)); in prepare_read_data()
1868 get_bvec_at(&con->v2.in_cursor, &bv); in prepare_read_data()
1869 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { in prepare_read_data()
1870 if (unlikely(!con->bounce_page)) { in prepare_read_data()
1871 con->bounce_page = alloc_page(GFP_NOIO); in prepare_read_data()
1872 if (!con->bounce_page) { in prepare_read_data()
1874 return -ENOMEM; in prepare_read_data()
1878 bv.bv_page = con->bounce_page; in prepare_read_data()
1882 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT; in prepare_read_data()
1890 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { in prepare_read_data_cont()
1891 con->in_data_crc = crc32c(con->in_data_crc, in prepare_read_data_cont()
1892 page_address(con->bounce_page), in prepare_read_data_cont()
1893 con->v2.in_bvec.bv_len); in prepare_read_data_cont()
1895 get_bvec_at(&con->v2.in_cursor, &bv); in prepare_read_data_cont()
1897 page_address(con->bounce_page), in prepare_read_data_cont()
1898 con->v2.in_bvec.bv_len); in prepare_read_data_cont()
1900 con->in_data_crc = ceph_crc32c_page(con->in_data_crc, in prepare_read_data_cont()
1901 con->v2.in_bvec.bv_page, in prepare_read_data_cont()
1902 con->v2.in_bvec.bv_offset, in prepare_read_data_cont()
1903 con->v2.in_bvec.bv_len); in prepare_read_data_cont()
1906 ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len); in prepare_read_data_cont()
1907 if (con->v2.in_cursor.total_resid) { in prepare_read_data_cont()
1908 get_bvec_at(&con->v2.in_cursor, &bv); in prepare_read_data_cont()
1909 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { in prepare_read_data_cont()
1910 bv.bv_page = con->bounce_page; in prepare_read_data_cont()
1914 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT); in prepare_read_data_cont()
1922 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); in prepare_read_data_cont()
1923 con->v2.in_state = IN_S_HANDLE_EPILOGUE; in prepare_read_data_cont()
1931 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor; in prepare_sparse_read_cont()
1933 WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT); in prepare_sparse_read_cont()
1935 if (iov_iter_is_bvec(&con->v2.in_iter)) { in prepare_sparse_read_cont()
1936 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { in prepare_sparse_read_cont()
1937 con->in_data_crc = crc32c(con->in_data_crc, in prepare_sparse_read_cont()
1938 page_address(con->bounce_page), in prepare_sparse_read_cont()
1939 con->v2.in_bvec.bv_len); in prepare_sparse_read_cont()
1942 page_address(con->bounce_page), in prepare_sparse_read_cont()
1943 con->v2.in_bvec.bv_len); in prepare_sparse_read_cont()
1945 con->in_data_crc = ceph_crc32c_page(con->in_data_crc, in prepare_sparse_read_cont()
1946 con->v2.in_bvec.bv_page, in prepare_sparse_read_cont()
1947 con->v2.in_bvec.bv_offset, in prepare_sparse_read_cont()
1948 con->v2.in_bvec.bv_len); in prepare_sparse_read_cont()
1951 ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len); in prepare_sparse_read_cont()
1952 cursor->sr_resid -= con->v2.in_bvec.bv_len; in prepare_sparse_read_cont()
1953 dout("%s: advance by 0x%x sr_resid 0x%x\n", __func__, in prepare_sparse_read_cont()
1954 con->v2.in_bvec.bv_len, cursor->sr_resid); in prepare_sparse_read_cont()
1955 WARN_ON_ONCE(cursor->sr_resid > cursor->total_resid); in prepare_sparse_read_cont()
1956 if (cursor->sr_resid) { in prepare_sparse_read_cont()
1958 if (bv.bv_len > cursor->sr_resid) in prepare_sparse_read_cont()
1959 bv.bv_len = cursor->sr_resid; in prepare_sparse_read_cont()
1960 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { in prepare_sparse_read_cont()
1961 bv.bv_page = con->bounce_page; in prepare_sparse_read_cont()
1965 con->v2.data_len_remain -= bv.bv_len; in prepare_sparse_read_cont()
1968 } else if (iov_iter_is_kvec(&con->v2.in_iter)) { in prepare_sparse_read_cont()
1970 if (con->v2.in_kvec_cnt) { in prepare_sparse_read_cont()
1971 WARN_ON_ONCE(con->v2.in_kvec_cnt > 1); in prepare_sparse_read_cont()
1972 con->in_data_crc = crc32c(con->in_data_crc, in prepare_sparse_read_cont()
1973 con->v2.in_kvecs[0].iov_base, in prepare_sparse_read_cont()
1974 con->v2.in_kvecs[0].iov_len); in prepare_sparse_read_cont()
1977 return -EIO; in prepare_sparse_read_cont()
1981 ret = con->ops->sparse_read(con, cursor, &buf); in prepare_sparse_read_cont()
1987 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); in prepare_sparse_read_cont()
1988 con->v2.in_state = IN_S_HANDLE_EPILOGUE; in prepare_sparse_read_cont()
1996 con->v2.data_len_remain -= ret; in prepare_sparse_read_cont()
2000 if (ret > cursor->total_resid) { in prepare_sparse_read_cont()
2002 __func__, ret, cursor->total_resid, cursor->resid); in prepare_sparse_read_cont()
2003 return -EIO; in prepare_sparse_read_cont()
2006 if (bv.bv_len > cursor->sr_resid) in prepare_sparse_read_cont()
2007 bv.bv_len = cursor->sr_resid; in prepare_sparse_read_cont()
2008 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { in prepare_sparse_read_cont()
2009 if (unlikely(!con->bounce_page)) { in prepare_sparse_read_cont()
2010 con->bounce_page = alloc_page(GFP_NOIO); in prepare_sparse_read_cont()
2011 if (!con->bounce_page) { in prepare_sparse_read_cont()
2013 return -ENOMEM; in prepare_sparse_read_cont()
2017 bv.bv_page = con->bounce_page; in prepare_sparse_read_cont()
2021 con->v2.data_len_remain -= ret; in prepare_sparse_read_cont()
2027 struct ceph_msg *msg = con->in_msg; in prepare_sparse_read_data()
2029 dout("%s: starting sparse read\n", __func__); in prepare_sparse_read_data()
2031 if (WARN_ON_ONCE(!con->ops->sparse_read)) in prepare_sparse_read_data()
2032 return -EOPNOTSUPP; in prepare_sparse_read_data()
2035 con->in_data_crc = -1; in prepare_sparse_read_data()
2037 ceph_msg_data_cursor_init(&con->v2.in_cursor, msg, in prepare_sparse_read_data()
2038 msg->sparse_read_total); in prepare_sparse_read_data()
2041 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT; in prepare_sparse_read_data()
2042 con->v2.data_len_remain = data_len(msg); in prepare_sparse_read_data()
2048 struct ceph_msg *msg = con->in_msg; in prepare_read_tail_plain()
2057 add_in_kvec(con, msg->front.iov_base, front_len(msg)); in prepare_read_tail_plain()
2058 WARN_ON(msg->front.iov_len != front_len(msg)); in prepare_read_tail_plain()
2061 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); in prepare_read_tail_plain()
2062 WARN_ON(msg->middle->vec.iov_len != middle_len(msg)); in prepare_read_tail_plain()
2066 if (msg->sparse_read_total) in prepare_read_tail_plain()
2067 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA; in prepare_read_tail_plain()
2069 con->v2.in_state = IN_S_PREPARE_READ_DATA; in prepare_read_tail_plain()
2071 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); in prepare_read_tail_plain()
2072 con->v2.in_state = IN_S_HANDLE_EPILOGUE; in prepare_read_tail_plain()
2081 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i, in prepare_read_enc_page()
2082 con->v2.in_enc_resid); in prepare_read_enc_page()
2083 WARN_ON(!con->v2.in_enc_resid); in prepare_read_enc_page()
2085 bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i], in prepare_read_enc_page()
2086 min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0); in prepare_read_enc_page()
2089 con->v2.in_enc_i++; in prepare_read_enc_page()
2090 con->v2.in_enc_resid -= bv.bv_len; in prepare_read_enc_page()
2092 if (con->v2.in_enc_resid) { in prepare_read_enc_page()
2093 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE; in prepare_read_enc_page()
2101 WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt); in prepare_read_enc_page()
2102 con->v2.in_state = IN_S_HANDLE_EPILOGUE; in prepare_read_enc_page()
2111 tail_len = tail_onwire_len(con->in_msg, true); in prepare_read_tail_secure()
2119 WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt); in prepare_read_tail_secure()
2120 con->v2.in_enc_pages = enc_pages; in prepare_read_tail_secure()
2121 con->v2.in_enc_page_cnt = enc_page_cnt; in prepare_read_tail_secure()
2122 con->v2.in_enc_resid = tail_len; in prepare_read_tail_secure()
2123 con->v2.in_enc_i = 0; in prepare_read_tail_secure()
2131 con->in_seq++; in __finish_skip()
2137 struct ceph_frame_desc *desc = &con->v2.in_desc; in prepare_skip_message()
2140 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1], in prepare_skip_message()
2141 desc->fd_lens[2], desc->fd_lens[3]); in prepare_skip_message()
2143 tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2], in prepare_skip_message()
2144 desc->fd_lens[3], con_secure(con)); in prepare_skip_message()
2149 con->v2.in_state = IN_S_FINISH_SKIP; in prepare_skip_message()
2158 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN); in process_banner_prefix()
2160 p = con->v2.in_kvecs[0].iov_base; in process_banner_prefix()
2163 con->error_msg = "server is speaking msgr1 protocol"; in process_banner_prefix()
2165 con->error_msg = "protocol error, bad banner"; in process_banner_prefix()
2166 return -EINVAL; in process_banner_prefix()
2171 dout("%s con %p payload_len %d\n", __func__, con, payload_len); in process_banner_prefix()
2178 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len; in process_banner_payload()
2185 p = con->v2.in_kvecs[0].iov_base; in process_banner_payload()
2189 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n", in process_banner_payload()
2195 con->error_msg = "missing required protocol features"; in process_banner_payload()
2196 return -EINVAL; in process_banner_payload()
2201 con->error_msg = "missing required protocol features"; in process_banner_payload()
2202 return -EINVAL; in process_banner_payload()
2212 con->state = CEPH_CON_S_V2_HELLO; in process_banner_payload()
2218 return -EINVAL; in process_banner_payload()
2223 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; in process_hello()
2228 if (con->state != CEPH_CON_S_V2_HELLO) { in process_hello()
2229 con->error_msg = "protocol error, unexpected hello"; in process_hello()
2230 return -EINVAL; in process_hello()
2240 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con, in process_hello()
2243 if (entity_type != con->peer_name.type) { in process_hello()
2245 con->peer_name.type, entity_type); in process_hello()
2246 con->error_msg = "wrong peer at address"; in process_hello()
2247 return -EINVAL; in process_hello()
2257 memcpy(&my_addr->in_addr, &addr_for_me.in_addr, in process_hello()
2258 sizeof(my_addr->in_addr)); in process_hello()
2260 dout("%s con %p set my addr %s, as seen by peer %s\n", in process_hello()
2262 ceph_pr_addr(&con->peer_addr)); in process_hello()
2264 dout("%s con %p my addr already set %s\n", in process_hello()
2269 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY); in process_hello()
2270 WARN_ON(!my_addr->nonce); in process_hello()
2275 if (ret != -EAGAIN) in process_hello()
2280 con->state = CEPH_CON_S_V2_AUTH; in process_hello()
2285 return -EINVAL; in process_hello()
2297 if (con->state != CEPH_CON_S_V2_AUTH) { in process_auth_bad_method()
2298 con->error_msg = "protocol error, unexpected auth_bad_method"; in process_auth_bad_method()
2299 return -EINVAL; in process_auth_bad_method()
2304 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto, in process_auth_bad_method()
2310 return -EINVAL; in process_auth_bad_method()
2314 dout("%s con %p allowed_protos[%d] %d\n", __func__, con, in process_auth_bad_method()
2321 return -EINVAL; in process_auth_bad_method()
2325 dout("%s con %p allowed_modes[%d] %d\n", __func__, con, in process_auth_bad_method()
2329 mutex_unlock(&con->mutex); in process_auth_bad_method()
2330 ret = con->ops->handle_auth_bad_method(con, used_proto, result, in process_auth_bad_method()
2335 mutex_lock(&con->mutex); in process_auth_bad_method()
2336 if (con->state != CEPH_CON_S_V2_AUTH) { in process_auth_bad_method()
2337 dout("%s con %p state changed to %d\n", __func__, con, in process_auth_bad_method()
2338 con->state); in process_auth_bad_method()
2339 return -EAGAIN; in process_auth_bad_method()
2342 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret); in process_auth_bad_method()
2347 return -EINVAL; in process_auth_bad_method()
2356 if (con->state != CEPH_CON_S_V2_AUTH) { in process_auth_reply_more()
2357 con->error_msg = "protocol error, unexpected auth_reply_more"; in process_auth_reply_more()
2358 return -EINVAL; in process_auth_reply_more()
2364 dout("%s con %p payload_len %d\n", __func__, con, payload_len); in process_auth_reply_more()
2369 if (ret != -EAGAIN) in process_auth_reply_more()
2378 return -EINVAL; in process_auth_reply_more()
2398 if (con->state != CEPH_CON_S_V2_AUTH) { in process_auth_done()
2399 con->error_msg = "protocol error, unexpected auth_done"; in process_auth_done()
2400 return -EINVAL; in process_auth_done()
2404 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad); in process_auth_done()
2407 dout("%s con %p global_id %llu con_mode %d payload_len %d\n", in process_auth_done()
2408 __func__, con, global_id, con->v2.con_mode, payload_len); in process_auth_done()
2410 mutex_unlock(&con->mutex); in process_auth_done()
2413 ret = con->ops->handle_auth_done(con, global_id, p, payload_len, in process_auth_done()
2416 mutex_lock(&con->mutex); in process_auth_done()
2417 if (con->state != CEPH_CON_S_V2_AUTH) { in process_auth_done()
2418 dout("%s con %p state changed to %d\n", __func__, con, in process_auth_done()
2419 con->state); in process_auth_done()
2420 ret = -EAGAIN; in process_auth_done()
2424 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret); in process_auth_done()
2440 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE; in process_auth_done()
2449 return -EINVAL; in process_auth_done()
2458 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) { in process_auth_signature()
2459 con->error_msg = "protocol error, unexpected auth_signature"; in process_auth_signature()
2460 return -EINVAL; in process_auth_signature()
2463 ret = hmac_sha256(con, con->v2.out_sign_kvecs, in process_auth_signature()
2464 con->v2.out_sign_kvec_cnt, hmac); in process_auth_signature()
2470 con->error_msg = "integrity error, bad auth signature"; in process_auth_signature()
2471 return -EBADMSG; in process_auth_signature()
2474 dout("%s con %p auth signature ok\n", __func__, con); in process_auth_signature()
2477 if (!con->v2.server_cookie) { in process_auth_signature()
2484 con->state = CEPH_CON_S_V2_SESSION_CONNECT; in process_auth_signature()
2492 con->state = CEPH_CON_S_V2_SESSION_RECONNECT; in process_auth_signature()
2499 return -EINVAL; in process_auth_signature()
2505 struct ceph_client *client = from_msgr(con->msgr); in process_server_ident()
2514 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { in process_server_ident()
2515 con->error_msg = "protocol error, unexpected server_ident"; in process_server_ident()
2516 return -EINVAL; in process_server_ident()
2532 …dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx… in process_server_ident()
2537 if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) { in process_server_ident()
2539 ceph_pr_addr(&con->peer_addr), in process_server_ident()
2540 le32_to_cpu(con->peer_addr.nonce), in process_server_ident()
2542 con->error_msg = "wrong peer at address"; in process_server_ident()
2543 return -EINVAL; in process_server_ident()
2546 if (client->required_features & ~features) { in process_server_ident()
2548 features, client->required_features & ~features); in process_server_ident()
2549 con->error_msg = "missing required protocol features"; in process_server_ident()
2550 return -EINVAL; in process_server_ident()
2554 * Both name->type and name->num are set in ceph_con_open() but in process_server_ident()
2555 * name->num may be bogus in the initial monmap. name->type is in process_server_ident()
2558 WARN_ON(!con->peer_name.type); in process_server_ident()
2559 con->peer_name.num = cpu_to_le64(global_id); in process_server_ident()
2560 con->v2.peer_global_seq = global_seq; in process_server_ident()
2561 con->peer_features = features; in process_server_ident()
2562 WARN_ON(required_features & ~client->supported_features); in process_server_ident()
2563 con->v2.server_cookie = cookie; in process_server_ident()
2567 WARN_ON(con->v2.server_cookie); in process_server_ident()
2569 WARN_ON(!con->v2.server_cookie); in process_server_ident()
2575 con->delay = 0; /* reset backoff memory */ in process_server_ident()
2577 con->state = CEPH_CON_S_OPEN; in process_server_ident()
2578 con->v2.out_state = OUT_S_GET_NEXT; in process_server_ident()
2583 return -EINVAL; in process_server_ident()
2589 struct ceph_client *client = from_msgr(con->msgr); in process_ident_missing_features()
2592 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { in process_ident_missing_features()
2593 con->error_msg = "protocol error, unexpected ident_missing_features"; in process_ident_missing_features()
2594 return -EINVAL; in process_ident_missing_features()
2599 client->supported_features, missing_features); in process_ident_missing_features()
2600 con->error_msg = "missing required protocol features"; in process_ident_missing_features()
2601 return -EINVAL; in process_ident_missing_features()
2605 return -EINVAL; in process_ident_missing_features()
2613 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { in process_session_reconnect_ok()
2614 con->error_msg = "protocol error, unexpected session_reconnect_ok"; in process_session_reconnect_ok()
2615 return -EINVAL; in process_session_reconnect_ok()
2620 dout("%s con %p seq %llu\n", __func__, con, seq); in process_session_reconnect_ok()
2626 con->delay = 0; /* reset backoff memory */ in process_session_reconnect_ok()
2628 con->state = CEPH_CON_S_OPEN; in process_session_reconnect_ok()
2629 con->v2.out_state = OUT_S_GET_NEXT; in process_session_reconnect_ok()
2634 return -EINVAL; in process_session_reconnect_ok()
2643 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { in process_session_retry()
2644 con->error_msg = "protocol error, unexpected session_retry"; in process_session_retry()
2645 return -EINVAL; in process_session_retry()
2650 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq); in process_session_retry()
2651 WARN_ON(connect_seq <= con->v2.connect_seq); in process_session_retry()
2652 con->v2.connect_seq = connect_seq + 1; in process_session_retry()
2667 return -EINVAL; in process_session_retry()
2676 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { in process_session_retry_global()
2677 con->error_msg = "protocol error, unexpected session_retry_global"; in process_session_retry_global()
2678 return -EINVAL; in process_session_retry_global()
2683 dout("%s con %p global_seq %llu\n", __func__, con, global_seq); in process_session_retry_global()
2684 WARN_ON(global_seq <= con->v2.global_seq); in process_session_retry_global()
2685 con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq); in process_session_retry_global()
2700 return -EINVAL; in process_session_retry_global()
2709 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { in process_session_reset()
2710 con->error_msg = "protocol error, unexpected session_reset"; in process_session_reset()
2711 return -EINVAL; in process_session_reset()
2716 con->error_msg = "protocol error, bad session_reset"; in process_session_reset()
2717 return -EINVAL; in process_session_reset()
2720 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name), in process_session_reset()
2721 ceph_pr_addr(&con->peer_addr)); in process_session_reset()
2724 mutex_unlock(&con->mutex); in process_session_reset()
2725 if (con->ops->peer_reset) in process_session_reset()
2726 con->ops->peer_reset(con); in process_session_reset()
2727 mutex_lock(&con->mutex); in process_session_reset()
2728 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { in process_session_reset()
2729 dout("%s con %p state changed to %d\n", __func__, con, in process_session_reset()
2730 con->state); in process_session_reset()
2731 return -EAGAIN; in process_session_reset()
2743 con->state = CEPH_CON_S_V2_SESSION_CONNECT; in process_session_reset()
2748 return -EINVAL; in process_session_reset()
2754 if (con->state != CEPH_CON_S_OPEN) { in process_keepalive2_ack()
2755 con->error_msg = "protocol error, unexpected keepalive2_ack"; in process_keepalive2_ack()
2756 return -EINVAL; in process_keepalive2_ack()
2760 ceph_decode_timespec64(&con->last_keepalive_ack, p); in process_keepalive2_ack()
2762 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, in process_keepalive2_ack()
2763 con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec); in process_keepalive2_ack()
2769 return -EINVAL; in process_keepalive2_ack()
2776 if (con->state != CEPH_CON_S_OPEN) { in process_ack()
2777 con->error_msg = "protocol error, unexpected ack"; in process_ack()
2778 return -EINVAL; in process_ack()
2783 dout("%s con %p seq %llu\n", __func__, con, seq); in process_ack()
2789 return -EINVAL; in process_ack()
2794 int tag = con->v2.in_desc.fd_tag; in process_control()
2797 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p)); in process_control()
2839 default: in process_control()
2841 con->error_msg = "protocol error, bad tag"; in process_control()
2842 return -EINVAL; in process_control()
2845 dout("%s con %p error %d\n", __func__, con, ret); in process_control()
2855 * 1 - con->in_msg set, read message
2856 * 0 - skip message
2857 * <0 - error
2862 struct ceph_frame_desc *desc = &con->v2.in_desc; in process_message_header()
2870 seq = le64_to_cpu(hdr2->seq); in process_message_header()
2871 if ((s64)seq - (s64)con->in_seq < 1) { in process_message_header()
2873 ENTITY_NAME(con->peer_name), in process_message_header()
2874 ceph_pr_addr(&con->peer_addr), in process_message_header()
2875 seq, con->in_seq + 1); in process_message_header()
2878 if ((s64)seq - (s64)con->in_seq > 1) { in process_message_header()
2879 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1); in process_message_header()
2880 con->error_msg = "bad message sequence # for incoming message"; in process_message_header()
2881 return -EBADE; in process_message_header()
2884 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq)); in process_message_header()
2886 fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2], in process_message_header()
2887 desc->fd_lens[3], &con->peer_name); in process_message_header()
2892 WARN_ON(!con->in_msg ^ skip); in process_message_header()
2896 WARN_ON(!con->in_msg); in process_message_header()
2897 WARN_ON(con->in_msg->con != con); in process_message_header()
2907 * ceph_con_process_message() temporarily drops con->mutex. in process_message()
2909 if (con->state != CEPH_CON_S_OPEN) { in process_message()
2910 dout("%s con %p state changed to %d\n", __func__, con, in process_message()
2911 con->state); in process_message()
2912 return -EAGAIN; in process_message()
2921 void *end = p + con->v2.in_desc.fd_lens[0]; in __handle_control()
2925 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE) in __handle_control()
2936 msg = con->in_msg; /* set in process_message_header() */ in __handle_control()
2938 WARN_ON(front_len(msg) > msg->front_alloc_len); in __handle_control()
2939 msg->front.iov_len = front_len(msg); in __handle_control()
2941 msg->front.iov_len = 0; in __handle_control()
2944 WARN_ON(middle_len(msg) > msg->middle->alloc_len); in __handle_control()
2945 msg->middle->vec.iov_len = middle_len(msg); in __handle_control()
2946 } else if (msg->middle) { in __handle_control()
2947 msg->middle->vec.iov_len = 0; in __handle_control()
2961 struct ceph_frame_desc *desc = &con->v2.in_desc; in handle_preamble()
2967 if (ret == -EBADMSG) in handle_preamble()
2968 con->error_msg = "integrity error, bad preamble auth tag"; in handle_preamble()
2973 ret = decode_preamble(con->v2.in_buf, desc); in handle_preamble()
2975 if (ret == -EBADMSG) in handle_preamble()
2976 con->error_msg = "integrity error, bad crc"; in handle_preamble()
2978 con->error_msg = "protocol error, bad preamble"; in handle_preamble()
2982 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__, in handle_preamble()
2983 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0], in handle_preamble()
2984 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]); in handle_preamble()
2989 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN) in handle_preamble()
2992 return __handle_control(con, CTRL_BODY(con->v2.in_buf)); in handle_preamble()
2997 int ctrl_len = con->v2.in_desc.fd_lens[0]; in handle_control()
3005 con->error_msg = "integrity error, bad crc"; in handle_control()
3009 if (con->state == CEPH_CON_S_V2_AUTH) { in handle_control()
3012 return -ENOMEM; in handle_control()
3014 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len); in handle_control()
3018 return __handle_control(con, con->v2.in_kvecs[0].iov_base); in handle_control()
3029 if (ret == -EBADMSG) in handle_control_remainder()
3030 con->error_msg = "integrity error, bad control remainder auth tag"; in handle_control_remainder()
3034 return __handle_control(con, con->v2.in_kvecs[0].iov_base - in handle_control_remainder()
3046 if (ret == -EBADMSG) in handle_epilogue()
3047 con->error_msg = "integrity error, bad epilogue auth tag"; in handle_epilogue()
3052 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL); in handle_epilogue()
3054 con->error_msg = "protocol error, bad epilogue"; in handle_epilogue()
3058 ret = decode_epilogue(con->v2.in_buf, &front_crc, in handle_epilogue()
3061 con->error_msg = "protocol error, bad epilogue"; in handle_epilogue()
3068 con->error_msg = "integrity error, bad crc"; in handle_epilogue()
3078 dout("%s con %p\n", __func__, con); in finish_skip()
3081 gcm_inc_nonce(&con->v2.in_gcm_nonce); in finish_skip()
3090 dout("%s con %p state %d in_state %d\n", __func__, con, con->state, in populate_in_iter()
3091 con->v2.in_state); in populate_in_iter()
3092 WARN_ON(iov_iter_count(&con->v2.in_iter)); in populate_in_iter()
3094 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) { in populate_in_iter()
3096 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) { in populate_in_iter()
3098 } else if ((con->state >= CEPH_CON_S_V2_HELLO && in populate_in_iter()
3099 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) || in populate_in_iter()
3100 con->state == CEPH_CON_S_OPEN) { in populate_in_iter()
3101 switch (con->v2.in_state) { in populate_in_iter()
3135 default: in populate_in_iter()
3136 WARN(1, "bad in_state %d", con->v2.in_state); in populate_in_iter()
3137 return -EINVAL; in populate_in_iter()
3140 WARN(1, "bad state %d", con->state); in populate_in_iter()
3141 return -EINVAL; in populate_in_iter()
3144 dout("%s con %p error %d\n", __func__, con, ret); in populate_in_iter()
3148 if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) in populate_in_iter()
3149 return -ENODATA; in populate_in_iter()
3150 dout("%s con %p populated %zu\n", __func__, con, in populate_in_iter()
3151 iov_iter_count(&con->v2.in_iter)); in populate_in_iter()
3159 dout("%s con %p state %d need %zu\n", __func__, con, con->state, in ceph_con_v2_try_read()
3160 iov_iter_count(&con->v2.in_iter)); in ceph_con_v2_try_read()
3162 if (con->state == CEPH_CON_S_PREOPEN) in ceph_con_v2_try_read()
3170 if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) in ceph_con_v2_try_read()
3171 return -ENODATA; in ceph_con_v2_try_read()
3180 if (ret && ret != -EAGAIN && !con->error_msg) in ceph_con_v2_try_read()
3181 con->error_msg = "read processing error"; in ceph_con_v2_try_read()
3191 con->v2.out_epil.data_crc = -1; in queue_data()
3192 ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg, in queue_data()
3193 data_len(con->out_msg)); in queue_data()
3195 get_bvec_at(&con->v2.out_cursor, &bv); in queue_data()
3197 con->v2.out_state = OUT_S_QUEUE_DATA_CONT; in queue_data()
3204 con->v2.out_epil.data_crc = ceph_crc32c_page( in queue_data_cont()
3205 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, in queue_data_cont()
3206 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len); in queue_data_cont()
3208 ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len); in queue_data_cont()
3209 if (con->v2.out_cursor.total_resid) { in queue_data_cont()
3210 get_bvec_at(&con->v2.out_cursor, &bv); in queue_data_cont()
3212 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT); in queue_data_cont()
3222 con->v2.out_state = OUT_S_FINISH_MESSAGE; in queue_data_cont()
3229 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i, in queue_enc_page()
3230 con->v2.out_enc_resid); in queue_enc_page()
3231 WARN_ON(!con->v2.out_enc_resid); in queue_enc_page()
3233 bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i], in queue_enc_page()
3234 min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0); in queue_enc_page()
3237 con->v2.out_enc_i++; in queue_enc_page()
3238 con->v2.out_enc_resid -= bv.bv_len; in queue_enc_page()
3240 if (con->v2.out_enc_resid) { in queue_enc_page()
3241 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE); in queue_enc_page()
3249 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt); in queue_enc_page()
3250 con->v2.out_state = OUT_S_FINISH_MESSAGE; in queue_enc_page()
3255 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero); in queue_zeros()
3257 if (con->v2.out_zero) { in queue_zeros()
3259 con->v2.out_zero -= con->v2.out_bvec.bv_len; in queue_zeros()
3260 con->v2.out_state = OUT_S_QUEUE_ZEROS; in queue_zeros()
3265 * We've zero-filled everything up to epilogue. Queue epilogue in queue_zeros()
3271 con->v2.out_state = OUT_S_FINISH_MESSAGE; in queue_zeros()
3276 dout("%s con %p msg %p\n", __func__, con, con->out_msg); in finish_message()
3279 if (con->v2.out_enc_pages) { in finish_message()
3280 WARN_ON(!con->v2.out_enc_page_cnt); in finish_message()
3281 ceph_release_page_vector(con->v2.out_enc_pages, in finish_message()
3282 con->v2.out_enc_page_cnt); in finish_message()
3283 con->v2.out_enc_pages = NULL; in finish_message()
3284 con->v2.out_enc_page_cnt = 0; in finish_message()
3287 if (con->out_msg) { in finish_message()
3288 ceph_msg_put(con->out_msg); in finish_message()
3289 con->out_msg = NULL; in finish_message()
3292 con->v2.out_state = OUT_S_GET_NEXT; in finish_message()
3299 dout("%s con %p state %d out_state %d\n", __func__, con, con->state, in populate_out_iter()
3300 con->v2.out_state); in populate_out_iter()
3301 WARN_ON(iov_iter_count(&con->v2.out_iter)); in populate_out_iter()
3303 if (con->state != CEPH_CON_S_OPEN) { in populate_out_iter()
3304 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX || in populate_out_iter()
3305 con->state > CEPH_CON_S_V2_SESSION_RECONNECT); in populate_out_iter()
3309 switch (con->v2.out_state) { in populate_out_iter()
3311 WARN_ON(!con->out_msg); in populate_out_iter()
3315 WARN_ON(!con->out_msg); in populate_out_iter()
3322 WARN_ON(con->out_msg); /* revoked */ in populate_out_iter()
3330 default: in populate_out_iter()
3331 WARN(1, "bad out_state %d", con->v2.out_state); in populate_out_iter()
3332 return -EINVAL; in populate_out_iter()
3335 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT); in populate_out_iter()
3342 } else if (!list_empty(&con->out_queue)) { in populate_out_iter()
3349 } else if (con->in_seq > con->in_seq_acked) { in populate_out_iter()
3360 if (WARN_ON(!iov_iter_count(&con->v2.out_iter))) in populate_out_iter()
3361 return -ENODATA; in populate_out_iter()
3362 dout("%s con %p populated %zu\n", __func__, con, in populate_out_iter()
3363 iov_iter_count(&con->v2.out_iter)); in populate_out_iter()
3367 WARN_ON(iov_iter_count(&con->v2.out_iter)); in populate_out_iter()
3368 dout("%s con %p nothing pending\n", __func__, con); in populate_out_iter()
3377 dout("%s con %p state %d have %zu\n", __func__, con, con->state, in ceph_con_v2_try_write()
3378 iov_iter_count(&con->v2.out_iter)); in ceph_con_v2_try_write()
3381 if (con->state == CEPH_CON_S_PREOPEN) { in ceph_con_v2_try_write()
3382 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2); in ceph_con_v2_try_write()
3389 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0); in ceph_con_v2_try_write()
3390 if (con->v2.server_cookie) in ceph_con_v2_try_write()
3391 con->v2.connect_seq++; in ceph_con_v2_try_write()
3396 con->error_msg = "connect error"; in ceph_con_v2_try_write()
3404 con->error_msg = "connect error"; in ceph_con_v2_try_write()
3411 con->error_msg = "connect error"; in ceph_con_v2_try_write()
3416 if (!iov_iter_count(&con->v2.out_iter)) { in ceph_con_v2_try_write()
3419 if (ret && ret != -EAGAIN && !con->error_msg) in ceph_con_v2_try_write()
3420 con->error_msg = "write processing error"; in ceph_con_v2_try_write()
3425 tcp_sock_set_cork(con->sock->sk, true); in ceph_con_v2_try_write()
3433 if (ret && ret != -EAGAIN && !con->error_msg) in ceph_con_v2_try_write()
3434 con->error_msg = "write processing error"; in ceph_con_v2_try_write()
3439 tcp_sock_set_cork(con->sock->sk, false); in ceph_con_v2_try_write()
3450 zero_len -= len; in crc32c_zeros()
3460 WARN_ON(!resid || resid > front_len(con->out_msg)); in prepare_zero_front()
3461 sent = front_len(con->out_msg) - resid; in prepare_zero_front()
3462 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); in prepare_zero_front()
3465 con->v2.out_epil.front_crc = in prepare_zero_front()
3466 crc32c(-1, con->out_msg->front.iov_base, sent); in prepare_zero_front()
3467 con->v2.out_epil.front_crc = in prepare_zero_front()
3468 crc32c_zeros(con->v2.out_epil.front_crc, resid); in prepare_zero_front()
3470 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid); in prepare_zero_front()
3473 con->v2.out_iter.count -= resid; in prepare_zero_front()
3481 WARN_ON(!resid || resid > middle_len(con->out_msg)); in prepare_zero_middle()
3482 sent = middle_len(con->out_msg) - resid; in prepare_zero_middle()
3483 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); in prepare_zero_middle()
3486 con->v2.out_epil.middle_crc = in prepare_zero_middle()
3487 crc32c(-1, con->out_msg->middle->vec.iov_base, sent); in prepare_zero_middle()
3488 con->v2.out_epil.middle_crc = in prepare_zero_middle()
3489 crc32c_zeros(con->v2.out_epil.middle_crc, resid); in prepare_zero_middle()
3491 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid); in prepare_zero_middle()
3494 con->v2.out_iter.count -= resid; in prepare_zero_middle()
3500 dout("%s con %p\n", __func__, con); in prepare_zero_data()
3501 con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg)); in prepare_zero_data()
3502 out_zero_add(con, data_len(con->out_msg)); in prepare_zero_data()
3510 WARN_ON(!data_len(con->out_msg)); in revoke_at_queue_data()
3511 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); in revoke_at_queue_data()
3512 resid = iov_iter_count(&con->v2.out_iter); in revoke_at_queue_data()
3514 boundary = front_len(con->out_msg) + middle_len(con->out_msg); in revoke_at_queue_data()
3516 resid -= boundary; in revoke_at_queue_data()
3518 dout("%s con %p was sending head\n", __func__, con); in revoke_at_queue_data()
3519 if (front_len(con->out_msg)) in revoke_at_queue_data()
3520 prepare_zero_front(con, front_len(con->out_msg)); in revoke_at_queue_data()
3521 if (middle_len(con->out_msg)) in revoke_at_queue_data()
3522 prepare_zero_middle(con, middle_len(con->out_msg)); in revoke_at_queue_data()
3524 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); in revoke_at_queue_data()
3525 con->v2.out_state = OUT_S_QUEUE_ZEROS; in revoke_at_queue_data()
3529 boundary = middle_len(con->out_msg); in revoke_at_queue_data()
3531 resid -= boundary; in revoke_at_queue_data()
3532 dout("%s con %p was sending front\n", __func__, con); in revoke_at_queue_data()
3534 if (middle_len(con->out_msg)) in revoke_at_queue_data()
3535 prepare_zero_middle(con, middle_len(con->out_msg)); in revoke_at_queue_data()
3542 dout("%s con %p was sending middle\n", __func__, con); in revoke_at_queue_data()
3552 WARN_ON(!data_len(con->out_msg)); in revoke_at_queue_data_cont()
3553 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter)); in revoke_at_queue_data_cont()
3554 resid = iov_iter_count(&con->v2.out_iter); in revoke_at_queue_data_cont()
3555 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len); in revoke_at_queue_data_cont()
3556 sent = con->v2.out_bvec.bv_len - resid; in revoke_at_queue_data_cont()
3557 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); in revoke_at_queue_data_cont()
3560 con->v2.out_epil.data_crc = ceph_crc32c_page( in revoke_at_queue_data_cont()
3561 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, in revoke_at_queue_data_cont()
3562 con->v2.out_bvec.bv_offset, sent); in revoke_at_queue_data_cont()
3563 ceph_msg_data_advance(&con->v2.out_cursor, sent); in revoke_at_queue_data_cont()
3565 WARN_ON(resid > con->v2.out_cursor.total_resid); in revoke_at_queue_data_cont()
3566 con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc, in revoke_at_queue_data_cont()
3567 con->v2.out_cursor.total_resid); in revoke_at_queue_data_cont()
3569 con->v2.out_iter.count -= resid; in revoke_at_queue_data_cont()
3570 out_zero_add(con, con->v2.out_cursor.total_resid); in revoke_at_queue_data_cont()
3579 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); in revoke_at_finish_message()
3580 resid = iov_iter_count(&con->v2.out_iter); in revoke_at_finish_message()
3582 if (!front_len(con->out_msg) && !middle_len(con->out_msg) && in revoke_at_finish_message()
3583 !data_len(con->out_msg)) { in revoke_at_finish_message()
3585 dout("%s con %p was sending head (empty message) - noop\n", in revoke_at_finish_message()
3590 boundary = front_len(con->out_msg) + middle_len(con->out_msg) + in revoke_at_finish_message()
3593 resid -= boundary; in revoke_at_finish_message()
3595 dout("%s con %p was sending head\n", __func__, con); in revoke_at_finish_message()
3596 if (front_len(con->out_msg)) in revoke_at_finish_message()
3597 prepare_zero_front(con, front_len(con->out_msg)); in revoke_at_finish_message()
3598 if (middle_len(con->out_msg)) in revoke_at_finish_message()
3599 prepare_zero_middle(con, middle_len(con->out_msg)); in revoke_at_finish_message()
3600 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; in revoke_at_finish_message()
3601 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); in revoke_at_finish_message()
3602 con->v2.out_state = OUT_S_QUEUE_ZEROS; in revoke_at_finish_message()
3606 boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN; in revoke_at_finish_message()
3608 resid -= boundary; in revoke_at_finish_message()
3609 dout("%s con %p was sending front\n", __func__, con); in revoke_at_finish_message()
3611 if (middle_len(con->out_msg)) in revoke_at_finish_message()
3612 prepare_zero_middle(con, middle_len(con->out_msg)); in revoke_at_finish_message()
3613 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; in revoke_at_finish_message()
3620 resid -= boundary; in revoke_at_finish_message()
3621 dout("%s con %p was sending middle\n", __func__, con); in revoke_at_finish_message()
3623 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; in revoke_at_finish_message()
3629 dout("%s con %p was sending epilogue - noop\n", __func__, con); in revoke_at_finish_message()
3634 WARN_ON(con->v2.out_zero); in ceph_con_v2_revoke()
3637 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE && in ceph_con_v2_revoke()
3638 con->v2.out_state != OUT_S_FINISH_MESSAGE); in ceph_con_v2_revoke()
3639 dout("%s con %p secure - noop\n", __func__, con); in ceph_con_v2_revoke()
3643 switch (con->v2.out_state) { in ceph_con_v2_revoke()
3653 default: in ceph_con_v2_revoke()
3654 WARN(1, "bad out_state %d", con->v2.out_state); in ceph_con_v2_revoke()
3665 WARN_ON(!data_len(con->in_msg)); in revoke_at_prepare_read_data()
3666 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); in revoke_at_prepare_read_data()
3667 resid = iov_iter_count(&con->v2.in_iter); in revoke_at_prepare_read_data()
3670 remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN; in revoke_at_prepare_read_data()
3671 dout("%s con %p resid %d remaining %d\n", __func__, con, resid, in revoke_at_prepare_read_data()
3673 con->v2.in_iter.count -= resid; in revoke_at_prepare_read_data()
3675 con->v2.in_state = IN_S_FINISH_SKIP; in revoke_at_prepare_read_data()
3684 WARN_ON(!data_len(con->in_msg)); in revoke_at_prepare_read_data_cont()
3685 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); in revoke_at_prepare_read_data_cont()
3686 resid = iov_iter_count(&con->v2.in_iter); in revoke_at_prepare_read_data_cont()
3687 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len); in revoke_at_prepare_read_data_cont()
3688 recved = con->v2.in_bvec.bv_len - resid; in revoke_at_prepare_read_data_cont()
3689 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid); in revoke_at_prepare_read_data_cont()
3692 ceph_msg_data_advance(&con->v2.in_cursor, recved); in revoke_at_prepare_read_data_cont()
3693 WARN_ON(resid > con->v2.in_cursor.total_resid); in revoke_at_prepare_read_data_cont()
3696 dout("%s con %p total_resid %zu remaining %d\n", __func__, con, in revoke_at_prepare_read_data_cont()
3697 con->v2.in_cursor.total_resid, remaining); in revoke_at_prepare_read_data_cont()
3698 con->v2.in_iter.count -= resid; in revoke_at_prepare_read_data_cont()
3699 set_in_skip(con, con->v2.in_cursor.total_resid + remaining); in revoke_at_prepare_read_data_cont()
3700 con->v2.in_state = IN_S_FINISH_SKIP; in revoke_at_prepare_read_data_cont()
3708 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); in revoke_at_prepare_read_enc_page()
3709 resid = iov_iter_count(&con->v2.in_iter); in revoke_at_prepare_read_enc_page()
3710 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len); in revoke_at_prepare_read_enc_page()
3712 dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid, in revoke_at_prepare_read_enc_page()
3713 con->v2.in_enc_resid); in revoke_at_prepare_read_enc_page()
3714 con->v2.in_iter.count -= resid; in revoke_at_prepare_read_enc_page()
3715 set_in_skip(con, resid + con->v2.in_enc_resid); in revoke_at_prepare_read_enc_page()
3716 con->v2.in_state = IN_S_FINISH_SKIP; in revoke_at_prepare_read_enc_page()
3725 WARN_ON(!data_len(con->in_msg)); in revoke_at_prepare_sparse_data()
3726 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); in revoke_at_prepare_sparse_data()
3727 resid = iov_iter_count(&con->v2.in_iter); in revoke_at_prepare_sparse_data()
3728 dout("%s con %p resid %d\n", __func__, con, resid); in revoke_at_prepare_sparse_data()
3730 remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain; in revoke_at_prepare_sparse_data()
3731 con->v2.in_iter.count -= resid; in revoke_at_prepare_sparse_data()
3733 con->v2.in_state = IN_S_FINISH_SKIP; in revoke_at_prepare_sparse_data()
3740 resid = iov_iter_count(&con->v2.in_iter); in revoke_at_handle_epilogue()
3743 dout("%s con %p resid %d\n", __func__, con, resid); in revoke_at_handle_epilogue()
3744 con->v2.in_iter.count -= resid; in revoke_at_handle_epilogue()
3746 con->v2.in_state = IN_S_FINISH_SKIP; in revoke_at_handle_epilogue()
3751 switch (con->v2.in_state) { in ceph_con_v2_revoke_incoming()
3768 default: in ceph_con_v2_revoke_incoming()
3769 WARN(1, "bad in_state %d", con->v2.in_state); in ceph_con_v2_revoke_incoming()
3776 return con->v2.peer_global_seq; in ceph_con_v2_opened()
3781 con->v2.client_cookie = 0; in ceph_con_v2_reset_session()
3782 con->v2.server_cookie = 0; in ceph_con_v2_reset_session()
3783 con->v2.global_seq = 0; in ceph_con_v2_reset_session()
3784 con->v2.connect_seq = 0; in ceph_con_v2_reset_session()
3785 con->v2.peer_global_seq = 0; in ceph_con_v2_reset_session()
3790 iov_iter_truncate(&con->v2.in_iter, 0); in ceph_con_v2_reset_protocol()
3791 iov_iter_truncate(&con->v2.out_iter, 0); in ceph_con_v2_reset_protocol()
3792 con->v2.out_zero = 0; in ceph_con_v2_reset_protocol()
3798 if (con->v2.in_enc_pages) { in ceph_con_v2_reset_protocol()
3799 WARN_ON(!con->v2.in_enc_page_cnt); in ceph_con_v2_reset_protocol()
3800 ceph_release_page_vector(con->v2.in_enc_pages, in ceph_con_v2_reset_protocol()
3801 con->v2.in_enc_page_cnt); in ceph_con_v2_reset_protocol()
3802 con->v2.in_enc_pages = NULL; in ceph_con_v2_reset_protocol()
3803 con->v2.in_enc_page_cnt = 0; in ceph_con_v2_reset_protocol()
3805 if (con->v2.out_enc_pages) { in ceph_con_v2_reset_protocol()
3806 WARN_ON(!con->v2.out_enc_page_cnt); in ceph_con_v2_reset_protocol()
3807 ceph_release_page_vector(con->v2.out_enc_pages, in ceph_con_v2_reset_protocol()
3808 con->v2.out_enc_page_cnt); in ceph_con_v2_reset_protocol()
3809 con->v2.out_enc_pages = NULL; in ceph_con_v2_reset_protocol()
3810 con->v2.out_enc_page_cnt = 0; in ceph_con_v2_reset_protocol()
3813 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN; in ceph_con_v2_reset_protocol()
3814 memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN); in ceph_con_v2_reset_protocol()
3815 memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN); in ceph_con_v2_reset_protocol()
3817 if (con->v2.hmac_tfm) { in ceph_con_v2_reset_protocol()
3818 crypto_free_shash(con->v2.hmac_tfm); in ceph_con_v2_reset_protocol()
3819 con->v2.hmac_tfm = NULL; in ceph_con_v2_reset_protocol()
3821 if (con->v2.gcm_req) { in ceph_con_v2_reset_protocol()
3822 aead_request_free(con->v2.gcm_req); in ceph_con_v2_reset_protocol()
3823 con->v2.gcm_req = NULL; in ceph_con_v2_reset_protocol()
3825 if (con->v2.gcm_tfm) { in ceph_con_v2_reset_protocol()
3826 crypto_free_aead(con->v2.gcm_tfm); in ceph_con_v2_reset_protocol()
3827 con->v2.gcm_tfm = NULL; in ceph_con_v2_reset_protocol()