206 	struct tipc_aead __rcu *aead[KEY_MAX + 1];  member
240 	struct tipc_aead *aead;  member
247 	struct tipc_aead *aead;  member
251 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead);
252 static inline void tipc_aead_put(struct tipc_aead *aead);
254 static int tipc_aead_users(struct tipc_aead __rcu *aead);
255 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim);
256 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim);
257 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val);
258 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead);
259 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
266 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
271 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
275 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
283 				  struct tipc_aead *aead, u8 pos,
296 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
383 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead)  in tipc_aead_get()  argument
388 	tmp = rcu_dereference(aead);  in tipc_aead_get()
396 static inline void tipc_aead_put(struct tipc_aead *aead)  in tipc_aead_put()  argument
398 	if (aead && refcount_dec_and_test(&aead->refcnt))  in tipc_aead_put()
399 		call_rcu(&aead->rcu, tipc_aead_free);  in tipc_aead_put()
408 	struct tipc_aead *aead = container_of(rp, struct tipc_aead, rcu);  in tipc_aead_free()  local
411 	if (aead->cloned) {  in tipc_aead_free()
412 		tipc_aead_put(aead->cloned);  in tipc_aead_free()
414 		head = *get_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
415 		put_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
426 	free_percpu(aead->tfm_entry);  in tipc_aead_free()
427 	kfree_sensitive(aead->key);  in tipc_aead_free()
428 	kfree(aead);  in tipc_aead_free()
431 static int tipc_aead_users(struct tipc_aead __rcu *aead)  in tipc_aead_users()  argument
437 	tmp = rcu_dereference(aead);  in tipc_aead_users()
445 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_inc()  argument
450 	tmp = rcu_dereference(aead);  in tipc_aead_users_inc()
456 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_dec()  argument
461 	tmp = rcu_dereference(aead);  in tipc_aead_users_dec()
463 		atomic_add_unless(&rcu_dereference(aead)->users, -1, lim);  in tipc_aead_users_dec()
467 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val)  in tipc_aead_users_set()  argument
473 	tmp = rcu_dereference(aead);  in tipc_aead_users_set()
488 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead)  in tipc_aead_tfm_next()  argument
493 	tfm_entry = get_cpu_ptr(aead->tfm_entry);  in tipc_aead_tfm_next()
514 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,  in tipc_aead_init()  argument
523 	if (unlikely(*aead))  in tipc_aead_init()
609 	*aead = tmp;  in tipc_aead_init()
628 	struct tipc_aead *aead;  in tipc_aead_clone()  local
640 	aead = kzalloc(sizeof(*aead), GFP_ATOMIC);  in tipc_aead_clone()
641 	if (unlikely(!aead))  in tipc_aead_clone()
644 	aead->tfm_entry = alloc_percpu_gfp(struct tipc_tfm *, GFP_ATOMIC);  in tipc_aead_clone()
645 	if (unlikely(!aead->tfm_entry)) {  in tipc_aead_clone()
646 		kfree_sensitive(aead);  in tipc_aead_clone()
651 		*per_cpu_ptr(aead->tfm_entry, cpu) =  in tipc_aead_clone()
655 	memcpy(aead->hint, src->hint, sizeof(src->hint));  in tipc_aead_clone()
656 	aead->mode = src->mode;  in tipc_aead_clone()
657 	aead->salt = src->salt;  in tipc_aead_clone()
658 	aead->authsize = src->authsize;  in tipc_aead_clone()
659 	atomic_set(&aead->users, 0);  in tipc_aead_clone()
660 	atomic64_set(&aead->seqno, 0);  in tipc_aead_clone()
661 	refcount_set(&aead->refcnt, 1);  in tipc_aead_clone()
664 	aead->cloned = src;  in tipc_aead_clone()
666 	*dst = aead;  in tipc_aead_clone()
732 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,  in tipc_aead_encrypt()  argument
737 	struct crypto_aead *tfm = tipc_aead_tfm_next(aead);  in tipc_aead_encrypt()
750 	tailen = len - skb->len + aead->authsize;  in tipc_aead_encrypt()
792 	salt = aead->salt;  in tipc_aead_encrypt()
793 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_encrypt()
810 	tx_ctx->aead = aead;  in tipc_aead_encrypt()
838 	struct tipc_aead *aead = tx_ctx->aead;  in tipc_aead_encrypt_done()  local
839 	struct tipc_crypto *tx = aead->crypto;  in tipc_aead_encrypt_done()
862 	tipc_aead_put(aead);  in tipc_aead_encrypt_done()
877 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,  in tipc_aead_decrypt()  argument
891 	if (unlikely(!aead))  in tipc_aead_decrypt()
901 	tfm = tipc_aead_tfm_next(aead);  in tipc_aead_decrypt()
917 	salt = aead->salt;  in tipc_aead_decrypt()
918 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_decrypt()
935 	rx_ctx->aead = aead;  in tipc_aead_decrypt()
962 	struct tipc_aead *aead = rx_ctx->aead;  in tipc_aead_decrypt_done()  local
963 	struct tipc_crypto_stats __percpu *stats = aead->crypto->stats;  in tipc_aead_decrypt_done()
964 	struct net *net = aead->crypto->net;  in tipc_aead_decrypt_done()
978 	tipc_crypto_rcv_complete(net, aead, b, &skb, err);  in tipc_aead_decrypt_done()
1030 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,  in tipc_ehdr_build()  argument
1049 	if (!__rx || aead->mode == CLUSTER_KEY)  in tipc_ehdr_build()
1050 		seqno = atomic64_inc_return(&aead->seqno);  in tipc_ehdr_build()
1069 	ehdr->master_key = aead->crypto->key_master;  in tipc_ehdr_build()
1122 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_init()  local
1126 	rc = tipc_aead_init(&aead, ukey, mode);  in tipc_crypto_key_init()
1130 		rc = tipc_crypto_key_attach(c, aead, 0, master_key);  in tipc_crypto_key_init()
1132 			tipc_aead_free(&aead->rcu);  in tipc_crypto_key_init()
1148 				  struct tipc_aead *aead, u8 pos,  in tipc_crypto_key_attach()  argument
1164 		if (tipc_aead_users(c->aead[key.pending]) > 0)  in tipc_crypto_key_attach()
1186 	aead->crypto = c;  in tipc_crypto_key_attach()
1187 	aead->gen = (is_tx(c)) ? ++c->key_gen : c->key_gen;  in tipc_crypto_key_attach()
1188 	tipc_aead_rcu_replace(c->aead[new_key], aead, &c->lock);  in tipc_crypto_key_attach()
1221 			tipc_aead_users_dec(tx->aead[k], 0);  in tipc_crypto_key_flush()
1230 		tipc_crypto_key_detach(c->aead[k], &c->lock);  in tipc_crypto_key_flush()
1266 	if (tipc_aead_users(rx->aead[key.pending]) > 0)  in tipc_crypto_key_try_align()
1270 	tmp1 = tipc_aead_rcu_ptr(rx->aead[key.pending], &rx->lock);  in tipc_crypto_key_try_align()
1273 	rcu_assign_pointer(rx->aead[key.pending], NULL);  in tipc_crypto_key_try_align()
1277 		tmp2 = rcu_replace_pointer(rx->aead[key.passive], tmp2, lockdep_is_held(&rx->lock));  in tipc_crypto_key_try_align()
1284 	rcu_assign_pointer(rx->aead[new_pending], tmp1);  in tipc_crypto_key_try_align()
1286 		rcu_assign_pointer(rx->aead[new_passive], tmp2);  in tipc_crypto_key_try_align()
1316 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_pick_tx()  local
1333 		aead = tipc_aead_rcu_ptr(tx->aead[KEY_MASTER], &tx->lock);  in tipc_crypto_key_pick_tx()
1341 		aead = tipc_aead_rcu_ptr(tx->aead[k], &tx->lock);  in tipc_crypto_key_pick_tx()
1342 		if (!aead)  in tipc_crypto_key_pick_tx()
1344 		if (aead->mode != CLUSTER_KEY ||  in tipc_crypto_key_pick_tx()
1345 		    aead == skb_cb->tx_clone_ctx.last) {  in tipc_crypto_key_pick_tx()
1346 			aead = NULL;  in tipc_crypto_key_pick_tx()
1350 		skb_cb->tx_clone_ctx.last = aead;  in tipc_crypto_key_pick_tx()
1359 	if (likely(aead))  in tipc_crypto_key_pick_tx()
1360 		WARN_ON(!refcount_inc_not_zero(&aead->refcnt));  in tipc_crypto_key_pick_tx()
1363 	return aead;  in tipc_crypto_key_pick_tx()
1426 			tipc_aead_users_inc(tx->aead[new], INT_MAX);  in tipc_crypto_key_synch()
1428 			tipc_aead_users_dec(tx->aead[cur], 0);  in tipc_crypto_key_synch()
1450 	tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_key_revoke()
1531 		tipc_aead_put(rcu_dereference(c->aead[k]));  in tipc_crypto_stop()
1552 	if (key.active && tipc_aead_users(tx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1554 	if (!key.pending || tipc_aead_users(tx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1561 		tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_timeout()
1571 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1585 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) > -10)  in tipc_crypto_timeout()
1589 	tipc_crypto_key_detach(rx->aead[key.pending], &rx->lock);  in tipc_crypto_timeout()
1598 	    tipc_aead_users(rx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1607 	tipc_aead_users_set(rx->aead[key.pending], 0);  in tipc_crypto_timeout()
1616 	    tipc_aead_users(rx->aead[key.passive]) > -10)  in tipc_crypto_timeout()
1620 	tipc_crypto_key_detach(rx->aead[key.passive], &rx->lock);  in tipc_crypto_timeout()
1689 	struct tipc_aead *aead = NULL;  in tipc_crypto_xmit()  local
1751 	aead = tipc_aead_get(tx->aead[tx_key]);  in tipc_crypto_xmit()
1752 	if (unlikely(!aead))  in tipc_crypto_xmit()
1754 	rc = tipc_ehdr_build(net, aead, tx_key, *skb, __rx);  in tipc_crypto_xmit()
1756 		rc = tipc_aead_encrypt(aead, *skb, b, dst, __dnode);  in tipc_crypto_xmit()
1779 	tipc_aead_put(aead);  in tipc_crypto_xmit()
1810 	struct tipc_aead *aead = NULL;  in tipc_crypto_rcv()  local
1835 	aead = tipc_crypto_key_pick_tx(tx, rx, *skb, tx_key);  in tipc_crypto_rcv()
1836 	if (aead)  in tipc_crypto_rcv()
1842 	if (!aead)  in tipc_crypto_rcv()
1843 		aead = tipc_aead_get(rx->aead[tx_key]);  in tipc_crypto_rcv()
1844 	rc = tipc_aead_decrypt(net, aead, *skb, b);  in tipc_crypto_rcv()
1870 					      rcu_access_pointer(rx->aead[n]));  in tipc_crypto_rcv()
1884 	tipc_crypto_rcv_complete(net, aead, b, skb, rc);  in tipc_crypto_rcv()
1888 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,  in tipc_crypto_rcv_complete()  argument
1893 	struct tipc_crypto *rx = aead->crypto;  in tipc_crypto_rcv_complete()
1899 	if (unlikely(is_tx(aead->crypto))) {  in tipc_crypto_rcv_complete()
1902 			 (rx) ? tipc_node_get_id_str(rx->node) : "-", err, aead,  in tipc_crypto_rcv_complete()
1906 			 aead->crypto->aead[1], aead->crypto->aead[2],  in tipc_crypto_rcv_complete()
1907 			 aead->crypto->aead[3]);  in tipc_crypto_rcv_complete()
1931 		if (tipc_aead_clone(&tmp, aead) < 0)  in tipc_crypto_rcv_complete()
1938 		tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
1939 		aead = tmp;  in tipc_crypto_rcv_complete()
1943 		tipc_aead_users_dec((struct tipc_aead __force __rcu *)aead, INT_MIN);  in tipc_crypto_rcv_complete()
1948 	tipc_aead_users_set((struct tipc_aead __force __rcu *)aead, 1);  in tipc_crypto_rcv_complete()
1963 	if (pskb_trim(*skb, (*skb)->len - aead->authsize))  in tipc_crypto_rcv_complete()
1993 	tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
2074 	struct tipc_aead *aead;  in tipc_crypto_key_dump()  local
2100 		aead = rcu_dereference(c->aead[k]);  in tipc_crypto_key_dump()
2101 		if (aead)  in tipc_crypto_key_dump()
2104 				       aead->hint,  in tipc_crypto_key_dump()
2105 				       (aead->mode == CLUSTER_KEY) ? "c" : "p",  in tipc_crypto_key_dump()
2106 				       atomic_read(&aead->users),  in tipc_crypto_key_dump()
2107 				       refcount_read(&aead->refcnt));  in tipc_crypto_key_dump()
2194 	struct tipc_aead *aead;  in tipc_crypto_key_distr()  local
2203 		aead = tipc_aead_get(tx->aead[key]);  in tipc_crypto_key_distr()
2204 		if (likely(aead)) {  in tipc_crypto_key_distr()
2205 			rc = tipc_crypto_key_xmit(tx->net, aead->key,  in tipc_crypto_key_distr()
2206 						  aead->gen, aead->mode,  in tipc_crypto_key_distr()
2208 			tipc_aead_put(aead);  in tipc_crypto_key_distr()
2441 	struct tipc_aead *aead;  in tipc_crypto_work_tx()  local
2449 	aead = rcu_dereference(tx->aead[key.active ?: KEY_MASTER]);  in tipc_crypto_work_tx()
2450 	if (unlikely(!aead)) {  in tipc_crypto_work_tx()
2457 	skey = kmemdup(aead->key, tipc_aead_key_size(aead->key), GFP_ATOMIC);  in tipc_crypto_work_tx()