314 static int rocker_desc_err(const struct rocker_desc_info *desc_info)  in rocker_desc_err()  argument
316 	int err = desc_info->desc->comp_err & ~ROCKER_DMA_DESC_COMP_ERR_GEN;  in rocker_desc_err()
342 static void rocker_desc_gen_clear(const struct rocker_desc_info *desc_info)  in rocker_desc_gen_clear()  argument
344 	desc_info->desc->comp_err &= ~ROCKER_DMA_DESC_COMP_ERR_GEN;  in rocker_desc_gen_clear()
347 static bool rocker_desc_gen(const struct rocker_desc_info *desc_info)  in rocker_desc_gen()  argument
349 	u32 comp_err = desc_info->desc->comp_err;  in rocker_desc_gen()
355 rocker_desc_cookie_ptr_get(const struct rocker_desc_info *desc_info)  in rocker_desc_cookie_ptr_get()  argument
357 	return (void *)(uintptr_t)desc_info->desc->cookie;  in rocker_desc_cookie_ptr_get()
360 static void rocker_desc_cookie_ptr_set(const struct rocker_desc_info *desc_info,  in rocker_desc_cookie_ptr_set()  argument
363 	desc_info->desc->cookie = (uintptr_t) ptr;  in rocker_desc_cookie_ptr_set()
369 	struct rocker_desc_info *desc_info;  in rocker_desc_head_get()  local
372 	desc_info = &info->desc_info[info->head];  in rocker_desc_head_get()
375 	desc_info->tlv_size = 0;  in rocker_desc_head_get()
376 	return desc_info;  in rocker_desc_head_get()
379 static void rocker_desc_commit(const struct rocker_desc_info *desc_info)  in rocker_desc_commit()  argument
381 	desc_info->desc->buf_size = desc_info->data_size;  in rocker_desc_commit()
382 	desc_info->desc->tlv_size = desc_info->tlv_size;  in rocker_desc_commit()
387 				 const struct rocker_desc_info *desc_info)  in rocker_desc_head_set()  argument
392 	rocker_desc_commit(desc_info);  in rocker_desc_head_set()
400 	struct rocker_desc_info *desc_info;  in rocker_desc_tail_get()  local
404 	desc_info = &info->desc_info[info->tail];  in rocker_desc_tail_get()
405 	if (!rocker_desc_gen(desc_info))  in rocker_desc_tail_get()
408 	desc_info->tlv_size = desc_info->desc->tlv_size;  in rocker_desc_tail_get()
409 	return desc_info;  in rocker_desc_tail_get()
438 	info->desc_info = kcalloc(info->size, sizeof(*info->desc_info),  in rocker_dma_ring_create()
440 	if (!info->desc_info)  in rocker_dma_ring_create()
447 		kfree(info->desc_info);  in rocker_dma_ring_create()
452 		info->desc_info[i].desc = &info->desc[i];  in rocker_dma_ring_create()
470 	kfree(info->desc_info);  in rocker_dma_ring_destroy()
484 		rocker_desc_head_set(rocker, info, &info->desc_info[i]);  in rocker_dma_ring_pass_to_producer()
485 	rocker_desc_commit(&info->desc_info[i]);  in rocker_dma_ring_pass_to_producer()
497 		struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_alloc()  local
516 		desc_info->data = buf;  in rocker_dma_ring_bufs_alloc()
517 		desc_info->data_size = buf_size;  in rocker_dma_ring_bufs_alloc()
518 		dma_unmap_addr_set(desc_info, mapaddr, dma_handle);  in rocker_dma_ring_bufs_alloc()
527 		const struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_alloc()  local
530 				 dma_unmap_addr(desc_info, mapaddr),  in rocker_dma_ring_bufs_alloc()
531 				 desc_info->data_size, direction);  in rocker_dma_ring_bufs_alloc()
532 		kfree(desc_info->data);  in rocker_dma_ring_bufs_alloc()
545 		const struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_free()  local
551 				 dma_unmap_addr(desc_info, mapaddr),  in rocker_dma_ring_bufs_free()
552 				 desc_info->data_size, direction);  in rocker_dma_ring_bufs_free()
553 		kfree(desc_info->data);  in rocker_dma_ring_bufs_free()
557 static int rocker_dma_cmd_ring_wait_alloc(struct rocker_desc_info *desc_info)  in rocker_dma_cmd_ring_wait_alloc()  argument
564 	rocker_desc_cookie_ptr_set(desc_info, wait);  in rocker_dma_cmd_ring_wait_alloc()
569 rocker_dma_cmd_ring_wait_free(const struct rocker_desc_info *desc_info)  in rocker_dma_cmd_ring_wait_free()  argument
571 	struct rocker_wait *wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_dma_cmd_ring_wait_free()
583 		err = rocker_dma_cmd_ring_wait_alloc(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_alloc()
591 		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_alloc()
601 		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_free()
673 				      struct rocker_desc_info *desc_info,  in rocker_dma_rx_ring_skb_map()  argument
684 	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_RX_FRAG_ADDR, dma_handle))  in rocker_dma_rx_ring_skb_map()
686 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_RX_FRAG_MAX_LEN, buf_len))  in rocker_dma_rx_ring_skb_map()
692 	desc_info->tlv_size = 0;  in rocker_dma_rx_ring_skb_map()
702 					struct rocker_desc_info *desc_info)  in rocker_dma_rx_ring_skb_alloc()  argument
712 	rocker_desc_cookie_ptr_set(desc_info, NULL);  in rocker_dma_rx_ring_skb_alloc()
713 	desc_info->tlv_size = 0;  in rocker_dma_rx_ring_skb_alloc()
718 	err = rocker_dma_rx_ring_skb_map(rocker_port, desc_info, skb, buf_len);  in rocker_dma_rx_ring_skb_alloc()
723 	rocker_desc_cookie_ptr_set(desc_info, skb);  in rocker_dma_rx_ring_skb_alloc()
743 					const struct rocker_desc_info *desc_info)  in rocker_dma_rx_ring_skb_free()  argument
746 	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_dma_rx_ring_skb_free()
750 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);  in rocker_dma_rx_ring_skb_free()
764 						   &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_alloc()
772 		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_alloc()
783 		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_free()
879 	const struct rocker_desc_info *desc_info;  in rocker_cmd_irq_handler()  local
884 	while ((desc_info = rocker_desc_tail_get(&rocker->cmd_ring))) {  in rocker_cmd_irq_handler()
885 		wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_cmd_irq_handler()
887 			rocker_desc_gen_clear(desc_info);  in rocker_cmd_irq_handler()
972 				const struct rocker_desc_info *desc_info)  in rocker_event_process()  argument
978 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_EVENT_MAX, desc_info);  in rocker_event_process()
1000 	const struct rocker_desc_info *desc_info;  in rocker_event_irq_handler()  local
1004 	while ((desc_info = rocker_desc_tail_get(&rocker->event_ring))) {  in rocker_event_irq_handler()
1005 		err = rocker_desc_err(desc_info);  in rocker_event_irq_handler()
1010 			err = rocker_event_process(rocker, desc_info);  in rocker_event_irq_handler()
1015 		rocker_desc_gen_clear(desc_info);  in rocker_event_irq_handler()
1016 		rocker_desc_head_set(rocker, &rocker->event_ring, desc_info);  in rocker_event_irq_handler()
1049 	struct rocker_desc_info *desc_info;  in rocker_cmd_exec()  local
1056 	desc_info = rocker_desc_head_get(&rocker->cmd_ring);  in rocker_cmd_exec()
1057 	if (!desc_info) {  in rocker_cmd_exec()
1062 	wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_cmd_exec()
1066 	err = prepare(rocker_port, desc_info, prepare_priv);  in rocker_cmd_exec()
1072 	rocker_desc_head_set(rocker, &rocker->cmd_ring, desc_info);  in rocker_cmd_exec()
1082 	err = rocker_desc_err(desc_info);  in rocker_cmd_exec()
1087 		err = process(rocker_port, desc_info, process_priv);  in rocker_cmd_exec()
1089 	rocker_desc_gen_clear(desc_info);  in rocker_cmd_exec()
1095 				  struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_prep()  argument
1100 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_get_port_settings_prep()
1103 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_get_port_settings_prep()
1106 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_get_port_settings_prep()
1109 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_get_port_settings_prep()
1115 					  const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_ethtool_proc()  argument
1125 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_ethtool_proc()
1154 					  const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_macaddr_proc()  argument
1162 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_macaddr_proc()
1181 				       const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_mode_proc()  argument
1189 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_mode_proc()
1210 					    const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_phys_name_proc()  argument
1220 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_phys_name_proc()
1251 					  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_ethtool_prep()  argument
1257 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_ethtool_prep()
1260 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_ethtool_prep()
1263 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_ethtool_prep()
1266 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_SPEED,  in rocker_cmd_set_port_settings_ethtool_prep()
1269 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX,  in rocker_cmd_set_port_settings_ethtool_prep()
1272 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG,  in rocker_cmd_set_port_settings_ethtool_prep()
1275 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_ethtool_prep()
1281 					  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_macaddr_prep()  argument
1287 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_macaddr_prep()
1290 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_macaddr_prep()
1293 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_macaddr_prep()
1296 	if (rocker_tlv_put(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR,  in rocker_cmd_set_port_settings_macaddr_prep()
1299 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_macaddr_prep()
1305 				      struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_mtu_prep()  argument
1311 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_mtu_prep()
1314 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_mtu_prep()
1317 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_mtu_prep()
1320 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MTU,  in rocker_cmd_set_port_settings_mtu_prep()
1323 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_mtu_prep()
1329 				  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_learning_prep()  argument
1335 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_learning_prep()
1338 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_learning_prep()
1341 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_learning_prep()
1344 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING,  in rocker_cmd_set_port_learning_prep()
1347 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_learning_prep()
1819 				       const struct rocker_desc_info *desc_info)  in rocker_tx_desc_frags_unmap()  argument
1827 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_TX_MAX, desc_info);  in rocker_tx_desc_frags_unmap()
1849 				       struct rocker_desc_info *desc_info,  in rocker_tx_desc_frag_map_put()  argument
1863 	frag = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAG);  in rocker_tx_desc_frag_map_put()
1866 	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_TX_FRAG_ATTR_ADDR,  in rocker_tx_desc_frag_map_put()
1869 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_TX_FRAG_ATTR_LEN,  in rocker_tx_desc_frag_map_put()
1872 	rocker_tlv_nest_end(desc_info, frag);  in rocker_tx_desc_frag_map_put()
1876 	rocker_tlv_nest_cancel(desc_info, frag);  in rocker_tx_desc_frag_map_put()
1886 	struct rocker_desc_info *desc_info;  in rocker_port_xmit()  local
1891 	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);  in rocker_port_xmit()
1892 	if (unlikely(!desc_info)) {  in rocker_port_xmit()
1898 	rocker_desc_cookie_ptr_set(desc_info, skb);  in rocker_port_xmit()
1900 	frags = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAGS);  in rocker_port_xmit()
1903 	err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,  in rocker_port_xmit()
1916 		err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,  in rocker_port_xmit()
1922 	rocker_tlv_nest_end(desc_info, frags);  in rocker_port_xmit()
1924 	rocker_desc_gen_clear(desc_info);  in rocker_port_xmit()
1925 	rocker_desc_head_set(rocker, &rocker_port->tx_ring, desc_info);  in rocker_port_xmit()
1927 	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);  in rocker_port_xmit()
1928 	if (!desc_info)  in rocker_port_xmit()
1934 	rocker_tx_desc_frags_unmap(rocker_port, desc_info);  in rocker_port_xmit()
1936 	rocker_tlv_nest_cancel(desc_info, frags);  in rocker_port_xmit()
2266 			       struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_stats_prep()  argument
2271 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_get_port_stats_prep()
2275 	cmd_stats = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_get_port_stats_prep()
2279 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_STATS_PPORT,  in rocker_cmd_get_port_stats_prep()
2283 	rocker_tlv_nest_end(desc_info, cmd_stats);  in rocker_cmd_get_port_stats_prep()
2290 				       const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_stats_ethtool_proc()  argument
2300 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_stats_ethtool_proc()
2381 	const struct rocker_desc_info *desc_info;  in rocker_port_poll_tx()  local
2386 	while ((desc_info = rocker_desc_tail_get(&rocker_port->tx_ring))) {  in rocker_port_poll_tx()
2389 		err = rocker_desc_err(desc_info);  in rocker_port_poll_tx()
2393 		rocker_tx_desc_frags_unmap(rocker_port, desc_info);  in rocker_port_poll_tx()
2395 		skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_port_poll_tx()
2418 			       struct rocker_desc_info *desc_info)  in rocker_port_rx_proc()  argument
2421 	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_port_rx_proc()
2428 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);  in rocker_port_rx_proc()
2448 	return rocker_dma_rx_ring_skb_alloc(rocker_port, desc_info);  in rocker_port_rx_proc()
2460 	struct rocker_desc_info *desc_info;  in rocker_port_poll_rx()  local
2466 	       (desc_info = rocker_desc_tail_get(&rocker_port->rx_ring))) {  in rocker_port_poll_rx()
2467 		err = rocker_desc_err(desc_info);  in rocker_port_poll_rx()
2474 						  desc_info);  in rocker_port_poll_rx()
2482 		rocker_desc_gen_clear(desc_info);  in rocker_port_poll_rx()
2483 		rocker_desc_head_set(rocker, &rocker_port->rx_ring, desc_info);  in rocker_port_poll_rx()