122 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
126 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
127 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
128 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
129 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
130 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
133 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
134 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
137 			      struct sk_msg *msg, u32 bytes);
139 			     struct sk_msg *msg, u32 bytes);
140 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
144 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)  in sk_msg_check_to_free()  argument
146 	WARN_ON(i == msg->sg.end && bytes);  in sk_msg_check_to_free()
179 #define sk_msg_iter_prev(msg, which)			\  argument
180 	sk_msg_iter_var_prev(msg->sg.which)
182 #define sk_msg_iter_next(msg, which)			\  argument
183 	sk_msg_iter_var_next(msg->sg.which)
185 static inline void sk_msg_init(struct sk_msg *msg)  in sk_msg_init()  argument
187 	BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS);  in sk_msg_init()
188 	memset(msg, 0, sizeof(*msg));  in sk_msg_init()
189 	sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);  in sk_msg_init()
209 static inline bool sk_msg_full(const struct sk_msg *msg)  in sk_msg_full()  argument
211 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS;  in sk_msg_full()
214 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)  in sk_msg_elem_used()  argument
216 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end);  in sk_msg_elem_used()
219 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)  in sk_msg_elem()  argument
221 	return &msg->sg.data[which];  in sk_msg_elem()
224 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)  in sk_msg_elem_cpy()  argument
226 	return msg->sg.data[which];  in sk_msg_elem_cpy()
229 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)  in sk_msg_page()  argument
231 	return sg_page(sk_msg_elem(msg, which));  in sk_msg_page()
234 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)  in sk_msg_to_ingress()  argument
236 	return msg->flags & BPF_F_INGRESS;  in sk_msg_to_ingress()
239 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)  in sk_msg_compute_data_pointers()  argument
241 	struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);  in sk_msg_compute_data_pointers()
243 	if (test_bit(msg->sg.start, msg->sg.copy)) {  in sk_msg_compute_data_pointers()
244 		msg->data = NULL;  in sk_msg_compute_data_pointers()
245 		msg->data_end = NULL;  in sk_msg_compute_data_pointers()
247 		msg->data = sg_virt(sge);  in sk_msg_compute_data_pointers()
248 		msg->data_end = msg->data + sge->length;  in sk_msg_compute_data_pointers()
252 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,  in sk_msg_page_add()  argument
258 	sge = sk_msg_elem(msg, msg->sg.end);  in sk_msg_page_add()
262 	__set_bit(msg->sg.end, msg->sg.copy);  in sk_msg_page_add()
263 	msg->sg.size += len;  in sk_msg_page_add()
264 	sk_msg_iter_next(msg, end);  in sk_msg_page_add()
267 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)  in sk_msg_sg_copy()  argument
271 			__set_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
273 			__clear_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
275 		if (i == msg->sg.end)  in sk_msg_sg_copy()
280 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_set()  argument
282 	sk_msg_sg_copy(msg, start, true);  in sk_msg_sg_copy_set()
285 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_clear()  argument
287 	sk_msg_sg_copy(msg, start, false);  in sk_msg_sg_copy_clear()
321 				      struct sk_msg *msg)  in sk_psock_queue_msg()  argument
325 		list_add_tail(&msg->list, &psock->ingress_msg);  in sk_psock_queue_msg()
327 		sk_msg_free(psock->sk, msg);  in sk_psock_queue_msg()
328 		kfree(msg);  in sk_psock_queue_msg()
335 	struct sk_msg *msg;  in sk_psock_dequeue_msg()  local
338 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_dequeue_msg()
339 	if (msg)  in sk_psock_dequeue_msg()
340 		list_del(&msg->list);  in sk_psock_dequeue_msg()
342 	return msg;  in sk_psock_dequeue_msg()
347 	struct sk_msg *msg;  in sk_psock_peek_msg()  local
350 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_peek_msg()
352 	return msg;  in sk_psock_peek_msg()
356 					       struct sk_msg *msg)  in sk_psock_next_msg()  argument
361 	if (list_is_last(&msg->list, &psock->ingress_msg))  in sk_psock_next_msg()
364 		ret = list_next_entry(msg, list);  in sk_psock_next_msg()
374 static inline void kfree_sk_msg(struct sk_msg *msg)  in kfree_sk_msg()  argument
376 	if (msg->skb)  in kfree_sk_msg()
377 		consume_skb(msg->skb);  in kfree_sk_msg()
378 	kfree(msg);  in kfree_sk_msg()
415 			 struct sk_msg *msg);