Lines Matching full:rm
67 struct rds_message *rm, *tmp; in rds_send_path_reset() local
71 rm = cp->cp_xmit_rm; in rds_send_path_reset()
77 rds_message_unmapped(rm); in rds_send_path_reset()
78 rds_message_put(rm); in rds_send_path_reset()
95 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_reset()
96 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_path_reset()
97 set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags); in rds_send_path_reset()
139 struct rds_message *rm; in rds_send_xmit() local
201 rm = cp->cp_xmit_rm; in rds_send_xmit()
203 if (!rm) { in rds_send_xmit()
218 if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) { in rds_send_xmit()
219 rm = rds_cong_update_alloc(conn); in rds_send_xmit()
220 if (IS_ERR(rm)) { in rds_send_xmit()
221 ret = PTR_ERR(rm); in rds_send_xmit()
224 rm->data.op_active = 1; in rds_send_xmit()
225 rm->m_inc.i_conn_path = cp; in rds_send_xmit()
226 rm->m_inc.i_conn = cp->cp_conn; in rds_send_xmit()
228 cp->cp_xmit_rm = rm; in rds_send_xmit()
238 if (!rm) { in rds_send_xmit()
254 rm = list_entry(cp->cp_send_queue.next, in rds_send_xmit()
257 rds_message_addref(rm); in rds_send_xmit()
263 list_move_tail(&rm->m_conn_item, in rds_send_xmit()
269 if (!rm) in rds_send_xmit()
279 if (test_bit(RDS_MSG_FLUSH, &rm->m_flags) || in rds_send_xmit()
280 (rm->rdma.op_active && in rds_send_xmit()
281 test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags))) { in rds_send_xmit()
283 if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) in rds_send_xmit()
284 list_move(&rm->m_conn_item, &to_be_dropped); in rds_send_xmit()
290 len = ntohl(rm->m_inc.i_hdr.h_len); in rds_send_xmit()
293 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_xmit()
305 cp->cp_xmit_rm = rm; in rds_send_xmit()
309 if (rm->rdma.op_active && !cp->cp_xmit_rdma_sent) { in rds_send_xmit()
310 rm->m_final_op = &rm->rdma; in rds_send_xmit()
314 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
315 ret = conn->c_trans->xmit_rdma(conn, &rm->rdma); in rds_send_xmit()
317 clear_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
318 wake_up_interruptible(&rm->m_flush_wait); in rds_send_xmit()
325 if (rm->atomic.op_active && !cp->cp_xmit_atomic_sent) { in rds_send_xmit()
326 rm->m_final_op = &rm->atomic; in rds_send_xmit()
330 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
331 ret = conn->c_trans->xmit_atomic(conn, &rm->atomic); in rds_send_xmit()
333 clear_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
334 wake_up_interruptible(&rm->m_flush_wait); in rds_send_xmit()
348 if (rm->data.op_nents == 0) { in rds_send_xmit()
352 ops_present = (rm->atomic.op_active || rm->rdma.op_active); in rds_send_xmit()
353 if (rm->atomic.op_active && !rm->atomic.op_silent) in rds_send_xmit()
355 if (rm->rdma.op_active && !rm->rdma.op_silent) in rds_send_xmit()
359 && !rm->m_rdma_cookie) in rds_send_xmit()
360 rm->data.op_active = 0; in rds_send_xmit()
363 if (rm->data.op_active && !cp->cp_xmit_data_sent) { in rds_send_xmit()
364 rm->m_final_op = &rm->data; in rds_send_xmit()
366 ret = conn->c_trans->xmit(conn, rm, in rds_send_xmit()
381 sg = &rm->data.op_sg[cp->cp_xmit_sg]; in rds_send_xmit()
392 rm->data.op_nents); in rds_send_xmit()
397 (cp->cp_xmit_sg == rm->data.op_nents)) in rds_send_xmit()
402 * A rm will only take multiple times through this loop in rds_send_xmit()
404 * none), then we're done with the rm. in rds_send_xmit()
406 if (!rm->data.op_active || cp->cp_xmit_data_sent) { in rds_send_xmit()
415 rds_message_put(rm); in rds_send_xmit()
427 list_for_each_entry(rm, &to_be_dropped, m_conn_item) in rds_send_xmit()
428 rds_message_put(rm); in rds_send_xmit()
472 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm) in rds_send_sndbuf_remove() argument
474 u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_sndbuf_remove()
485 static inline int rds_send_is_acked(struct rds_message *rm, u64 ack, in rds_send_is_acked() argument
489 return is_acked(rm, ack); in rds_send_is_acked()
490 return be64_to_cpu(rm->m_inc.i_hdr.h_sequence) <= ack; in rds_send_is_acked()
499 void rds_rdma_send_complete(struct rds_message *rm, int status) in rds_rdma_send_complete() argument
506 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
508 ro = &rm->rdma; in rds_rdma_send_complete()
509 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) && in rds_rdma_send_complete()
512 rs = rm->m_rs; in rds_rdma_send_complete()
523 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
535 void rds_atomic_send_complete(struct rds_message *rm, int status) in rds_atomic_send_complete() argument
542 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
544 ao = &rm->atomic; in rds_atomic_send_complete()
545 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) in rds_atomic_send_complete()
548 rs = rm->m_rs; in rds_atomic_send_complete()
559 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
574 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status) in __rds_send_complete() argument
579 ro = &rm->rdma; in __rds_send_complete()
586 ao = &rm->atomic; in __rds_send_complete()
608 struct rds_message *rm; in rds_send_remove_from_sock() local
613 rm = list_entry(messages->next, struct rds_message, in rds_send_remove_from_sock()
615 list_del_init(&rm->m_conn_item); in rds_send_remove_from_sock()
623 * The message spinlock makes sure nobody clears rm->m_rs in rds_send_remove_from_sock()
627 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
628 if (!test_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) in rds_send_remove_from_sock()
631 if (rs != rm->m_rs) { in rds_send_remove_from_sock()
636 rs = rm->m_rs; in rds_send_remove_from_sock()
644 if (test_and_clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) { in rds_send_remove_from_sock()
645 struct rm_rdma_op *ro = &rm->rdma; in rds_send_remove_from_sock()
648 list_del_init(&rm->m_sock_item); in rds_send_remove_from_sock()
649 rds_send_sndbuf_remove(rs, rm); in rds_send_remove_from_sock()
658 rm->rdma.op_notifier = NULL; in rds_send_remove_from_sock()
665 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
666 rds_message_put(rm); in rds_send_remove_from_sock()
668 rds_message_put(rm); in rds_send_remove_from_sock()
688 struct rds_message *rm, *tmp; in rds_send_path_drop_acked() local
694 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_drop_acked()
695 if (!rds_send_is_acked(rm, ack, is_acked)) in rds_send_path_drop_acked()
698 list_move(&rm->m_conn_item, &list); in rds_send_path_drop_acked()
699 clear_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_path_drop_acked()
723 struct rds_message *rm, *tmp; in rds_send_drop_to() local
732 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) { in rds_send_drop_to()
734 (!ipv6_addr_equal(&dest->sin6_addr, &rm->m_daddr) || in rds_send_drop_to()
735 dest->sin6_port != rm->m_inc.i_hdr.h_dport)) in rds_send_drop_to()
738 list_move(&rm->m_sock_item, &list); in rds_send_drop_to()
739 rds_send_sndbuf_remove(rs, rm); in rds_send_drop_to()
740 clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_drop_to()
752 list_for_each_entry(rm, &list, m_sock_item) { in rds_send_drop_to()
754 conn = rm->m_inc.i_conn; in rds_send_drop_to()
756 cp = rm->m_inc.i_conn_path; in rds_send_drop_to()
762 * Maybe someone else beat us to removing rm from the conn. in rds_send_drop_to()
766 if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) { in rds_send_drop_to()
770 list_del_init(&rm->m_conn_item); in rds_send_drop_to()
777 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
780 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
783 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
785 rds_message_put(rm); in rds_send_drop_to()
791 rm = list_entry(list.next, struct rds_message, m_sock_item); in rds_send_drop_to()
792 list_del_init(&rm->m_sock_item); in rds_send_drop_to()
793 rds_message_wait(rm); in rds_send_drop_to()
800 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
803 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
806 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
808 rds_message_put(rm); in rds_send_drop_to()
819 struct rds_message *rm, __be16 sport, in rds_send_queue_rm() argument
828 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_queue_rm()
851 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_queue_rm()
853 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue); in rds_send_queue_rm()
854 set_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_queue_rm()
855 rds_message_addref(rm); in rds_send_queue_rm()
857 rm->m_rs = rs; in rds_send_queue_rm()
861 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0); in rds_send_queue_rm()
862 rm->m_inc.i_conn = conn; in rds_send_queue_rm()
863 rm->m_inc.i_conn_path = cp; in rds_send_queue_rm()
864 rds_message_addref(rm); in rds_send_queue_rm()
867 rm->m_inc.i_hdr.h_sequence = cpu_to_be64(cp->cp_next_tx_seq++); in rds_send_queue_rm()
868 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_queue_rm()
869 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_queue_rm()
873 rm, len, rs, rs->rs_snd_bytes, in rds_send_queue_rm()
874 (unsigned long long)be64_to_cpu(rm->m_inc.i_hdr.h_sequence)); in rds_send_queue_rm()
970 static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_zcopy() argument
976 !rm->data.op_mmp_znotifier) in rds_cmsg_zcopy()
979 rm->data.op_mmp_znotifier->z_cookie = *cookie; in rds_cmsg_zcopy()
983 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_send() argument
998 * rm->rdma.m_rdma_cookie and rm->rdma.m_rdma_mr. in rds_cmsg_send()
1004 ret = rds_cmsg_rdma_args(rs, rm, cmsg, &vct->vec[ind]); in rds_cmsg_send()
1009 ret = rds_cmsg_rdma_dest(rs, rm, cmsg); in rds_cmsg_send()
1013 ret = rds_cmsg_rdma_map(rs, rm, cmsg); in rds_cmsg_send()
1026 ret = rds_cmsg_atomic(rs, rm, cmsg); in rds_cmsg_send()
1030 ret = rds_cmsg_zcopy(rs, rm, cmsg); in rds_cmsg_send()
1107 struct rds_message *rm = NULL; in rds_sendmsg() local
1262 /* size of rm including all sgs */ in rds_sendmsg()
1267 rm = rds_message_alloc(ret, GFP_KERNEL); in rds_sendmsg()
1268 if (!rm) { in rds_sendmsg()
1273 /* Attach data to the rm */ in rds_sendmsg()
1275 rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs); in rds_sendmsg()
1276 if (IS_ERR(rm->data.op_sg)) { in rds_sendmsg()
1277 ret = PTR_ERR(rm->data.op_sg); in rds_sendmsg()
1280 ret = rds_message_copy_from_user(rm, &msg->msg_iter, zcopy); in rds_sendmsg()
1284 rm->data.op_active = 1; in rds_sendmsg()
1286 rm->m_daddr = daddr; in rds_sendmsg()
1311 rm->m_conn_path = cpath; in rds_sendmsg()
1314 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr, &vct); in rds_sendmsg()
1318 if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) { in rds_sendmsg()
1320 &rm->rdma, conn->c_trans->xmit_rdma); in rds_sendmsg()
1325 if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) { in rds_sendmsg()
1327 &rm->atomic, conn->c_trans->xmit_atomic); in rds_sendmsg()
1345 while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port, in rds_sendmsg()
1355 rds_send_queue_rm(rs, conn, cpath, rm, in rds_sendmsg()
1388 rds_message_put(rm); in rds_sendmsg()
1405 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1); in rds_sendmsg()
1407 if (rm) in rds_sendmsg()
1408 rds_message_put(rm); in rds_sendmsg()
1424 struct rds_message *rm; in rds_send_probe() local
1428 rm = rds_message_alloc(0, GFP_ATOMIC); in rds_send_probe()
1429 if (!rm) { in rds_send_probe()
1434 rm->m_daddr = cp->cp_conn->c_faddr; in rds_send_probe()
1435 rm->data.op_active = 1; in rds_send_probe()
1444 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_probe()
1445 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_probe()
1446 rds_message_addref(rm); in rds_send_probe()
1447 rm->m_inc.i_conn = cp->cp_conn; in rds_send_probe()
1448 rm->m_inc.i_conn_path = cp; in rds_send_probe()
1450 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, in rds_send_probe()
1452 rm->m_inc.i_hdr.h_flags |= h_flags; in rds_send_probe()
1460 rds_message_add_extension(&rm->m_inc.i_hdr, in rds_send_probe()
1463 rds_message_add_extension(&rm->m_inc.i_hdr, in rds_send_probe()
1479 rds_message_put(rm); in rds_send_probe()
1483 if (rm) in rds_send_probe()
1484 rds_message_put(rm); in rds_send_probe()