1 /*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Atul Gupta (atul.gupta@chelsio.com)
36 */
37
38 #define pr_fmt(fmt) "ch_ipsec: " fmt
39
40 #include <linux/kernel.h>
41 #include <linux/module.h>
42 #include <linux/skbuff.h>
43 #include <linux/rtnetlink.h>
44 #include <linux/highmem.h>
45 #include <linux/if_vlan.h>
46 #include <linux/ip.h>
47 #include <linux/netdevice.h>
48 #include <net/esp.h>
49 #include <net/xfrm.h>
50 #include <crypto/aes.h>
51 #include <crypto/hash.h>
52 #include <crypto/sha1.h>
53 #include <crypto/sha2.h>
54 #include <crypto/authenc.h>
55 #include <crypto/internal/aead.h>
56 #include <crypto/null.h>
57 #include <crypto/internal/skcipher.h>
58 #include <crypto/aead.h>
59 #include <crypto/scatterwalk.h>
60 #include <crypto/internal/hash.h>
61
62 #include "chcr_ipsec.h"
63
64 /*
65 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
66 * into a WR.
67 */
68 #define MAX_IMM_TX_PKT_LEN 256
69 #define GCM_ESP_IV_SIZE 8
70
71 static LIST_HEAD(uld_ctx_list);
72 static DEFINE_MUTEX(dev_mutex);
73
74 static bool ch_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
75 static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state);
76 static int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev);
77 static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop);
78 static void ch_ipsec_advance_esn_state(struct xfrm_state *x);
79 static void ch_ipsec_xfrm_free_state(struct xfrm_state *x);
80 static void ch_ipsec_xfrm_del_state(struct xfrm_state *x);
81 static int ch_ipsec_xfrm_add_state(struct xfrm_state *x,
82 struct netlink_ext_ack *extack);
83
84 static const struct xfrmdev_ops ch_ipsec_xfrmdev_ops = {
85 .xdo_dev_state_add = ch_ipsec_xfrm_add_state,
86 .xdo_dev_state_delete = ch_ipsec_xfrm_del_state,
87 .xdo_dev_state_free = ch_ipsec_xfrm_free_state,
88 .xdo_dev_offload_ok = ch_ipsec_offload_ok,
89 .xdo_dev_state_advance_esn = ch_ipsec_advance_esn_state,
90 };
91
92 static struct cxgb4_uld_info ch_ipsec_uld_info = {
93 .name = CHIPSEC_DRV_MODULE_NAME,
94 .add = ch_ipsec_uld_add,
95 .state_change = ch_ipsec_uld_state_change,
96 .tx_handler = ch_ipsec_xmit,
97 .xfrmdev_ops = &ch_ipsec_xfrmdev_ops,
98 };
99
ch_ipsec_uld_add(const struct cxgb4_lld_info * infop)100 static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop)
101 {
102 struct ipsec_uld_ctx *u_ctx;
103
104 pr_info_once("%s - version %s\n", CHIPSEC_DRV_DESC,
105 CHIPSEC_DRV_VERSION);
106 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
107 if (!u_ctx) {
108 u_ctx = ERR_PTR(-ENOMEM);
109 goto out;
110 }
111 u_ctx->lldi = *infop;
112 out:
113 return u_ctx;
114 }
115
ch_ipsec_uld_state_change(void * handle,enum cxgb4_state new_state)116 static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state)
117 {
118 struct ipsec_uld_ctx *u_ctx = handle;
119
120 pr_debug("new_state %u\n", new_state);
121 switch (new_state) {
122 case CXGB4_STATE_UP:
123 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
124 mutex_lock(&dev_mutex);
125 list_add_tail(&u_ctx->entry, &uld_ctx_list);
126 mutex_unlock(&dev_mutex);
127 break;
128 case CXGB4_STATE_START_RECOVERY:
129 case CXGB4_STATE_DOWN:
130 case CXGB4_STATE_DETACH:
131 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
132 list_del(&u_ctx->entry);
133 break;
134 default:
135 break;
136 }
137
138 return 0;
139 }
140
ch_ipsec_setauthsize(struct xfrm_state * x,struct ipsec_sa_entry * sa_entry)141 static int ch_ipsec_setauthsize(struct xfrm_state *x,
142 struct ipsec_sa_entry *sa_entry)
143 {
144 int hmac_ctrl;
145 int authsize = x->aead->alg_icv_len / 8;
146
147 sa_entry->authsize = authsize;
148
149 switch (authsize) {
150 case ICV_8:
151 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
152 break;
153 case ICV_12:
154 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
155 break;
156 case ICV_16:
157 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
158 break;
159 default:
160 return -EINVAL;
161 }
162 return hmac_ctrl;
163 }
164
ch_ipsec_setkey(struct xfrm_state * x,struct ipsec_sa_entry * sa_entry)165 static int ch_ipsec_setkey(struct xfrm_state *x,
166 struct ipsec_sa_entry *sa_entry)
167 {
168 int keylen = (x->aead->alg_key_len + 7) / 8;
169 unsigned char *key = x->aead->alg_key;
170 int ck_size, key_ctx_size = 0;
171 unsigned char ghash_h[AEAD_H_SIZE];
172 struct crypto_aes_ctx aes;
173 int ret = 0;
174
175 if (keylen > 3) {
176 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
177 memcpy(sa_entry->salt, key + keylen, 4);
178 }
179
180 if (keylen == AES_KEYSIZE_128) {
181 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
182 } else if (keylen == AES_KEYSIZE_192) {
183 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
184 } else if (keylen == AES_KEYSIZE_256) {
185 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
186 } else {
187 pr_err("GCM: Invalid key length %d\n", keylen);
188 ret = -EINVAL;
189 goto out;
190 }
191
192 memcpy(sa_entry->key, key, keylen);
193 sa_entry->enckey_len = keylen;
194 key_ctx_size = sizeof(struct _key_ctx) +
195 ((DIV_ROUND_UP(keylen, 16)) << 4) +
196 AEAD_H_SIZE;
197
198 sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
199 CHCR_KEYCTX_MAC_KEY_SIZE_128,
200 0, 0,
201 key_ctx_size >> 4);
202
203 /* Calculate the H = CIPH(K, 0 repeated 16 times).
204 * It will go in key context
205 */
206 ret = aes_expandkey(&aes, key, keylen);
207 if (ret) {
208 sa_entry->enckey_len = 0;
209 goto out;
210 }
211 memset(ghash_h, 0, AEAD_H_SIZE);
212 aes_encrypt(&aes, ghash_h, ghash_h);
213 memzero_explicit(&aes, sizeof(aes));
214
215 memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
216 16), ghash_h, AEAD_H_SIZE);
217 sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
218 AEAD_H_SIZE;
219 out:
220 return ret;
221 }
222
223 /*
224 * ch_ipsec_xfrm_add_state
225 * returns 0 on success, negative error if failed to send message to FPGA
226 * positive error if FPGA returned a bad response
227 */
ch_ipsec_xfrm_add_state(struct xfrm_state * x,struct netlink_ext_ack * extack)228 static int ch_ipsec_xfrm_add_state(struct xfrm_state *x,
229 struct netlink_ext_ack *extack)
230 {
231 struct ipsec_sa_entry *sa_entry;
232 int res = 0;
233
234 if (x->props.aalgo != SADB_AALG_NONE) {
235 NL_SET_ERR_MSG_MOD(extack, "Cannot offload authenticated xfrm states");
236 return -EINVAL;
237 }
238 if (x->props.calgo != SADB_X_CALG_NONE) {
239 NL_SET_ERR_MSG_MOD(extack, "Cannot offload compressed xfrm states");
240 return -EINVAL;
241 }
242 if (x->props.family != AF_INET &&
243 x->props.family != AF_INET6) {
244 NL_SET_ERR_MSG_MOD(extack, "Only IPv4/6 xfrm state offloaded");
245 return -EINVAL;
246 }
247 if (x->props.mode != XFRM_MODE_TRANSPORT &&
248 x->props.mode != XFRM_MODE_TUNNEL) {
249 NL_SET_ERR_MSG_MOD(extack, "Only transport and tunnel xfrm offload");
250 return -EINVAL;
251 }
252 if (x->id.proto != IPPROTO_ESP) {
253 NL_SET_ERR_MSG_MOD(extack, "Only ESP xfrm state offloaded");
254 return -EINVAL;
255 }
256 if (x->encap) {
257 NL_SET_ERR_MSG_MOD(extack, "Encapsulated xfrm state not offloaded");
258 return -EINVAL;
259 }
260 if (!x->aead) {
261 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without aead");
262 return -EINVAL;
263 }
264 if (x->aead->alg_icv_len != 128 &&
265 x->aead->alg_icv_len != 96) {
266 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with AEAD ICV length other than 96b & 128b");
267 return -EINVAL;
268 }
269 if ((x->aead->alg_key_len != 128 + 32) &&
270 (x->aead->alg_key_len != 256 + 32)) {
271 NL_SET_ERR_MSG_MOD(extack, "cannot offload xfrm states with AEAD key length other than 128/256 bit");
272 return -EINVAL;
273 }
274 if (x->tfcpad) {
275 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with tfc padding");
276 return -EINVAL;
277 }
278 if (!x->geniv) {
279 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without geniv");
280 return -EINVAL;
281 }
282 if (strcmp(x->geniv, "seqiv")) {
283 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with geniv other than seqiv");
284 return -EINVAL;
285 }
286 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
287 NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload");
288 return -EINVAL;
289 }
290
291 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
292 if (!sa_entry) {
293 res = -ENOMEM;
294 goto out;
295 }
296
297 sa_entry->hmac_ctrl = ch_ipsec_setauthsize(x, sa_entry);
298 if (x->props.flags & XFRM_STATE_ESN)
299 sa_entry->esn = 1;
300 ch_ipsec_setkey(x, sa_entry);
301 x->xso.offload_handle = (unsigned long)sa_entry;
302 try_module_get(THIS_MODULE);
303 out:
304 return res;
305 }
306
ch_ipsec_xfrm_del_state(struct xfrm_state * x)307 static void ch_ipsec_xfrm_del_state(struct xfrm_state *x)
308 {
309 /* do nothing */
310 if (!x->xso.offload_handle)
311 return;
312 }
313
ch_ipsec_xfrm_free_state(struct xfrm_state * x)314 static void ch_ipsec_xfrm_free_state(struct xfrm_state *x)
315 {
316 struct ipsec_sa_entry *sa_entry;
317
318 if (!x->xso.offload_handle)
319 return;
320
321 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
322 kfree(sa_entry);
323 module_put(THIS_MODULE);
324 }
325
ch_ipsec_offload_ok(struct sk_buff * skb,struct xfrm_state * x)326 static bool ch_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
327 {
328 if (x->props.family == AF_INET) {
329 /* Offload with IP options is not supported yet */
330 if (ip_hdr(skb)->ihl > 5)
331 return false;
332 } else {
333 /* Offload with IPv6 extension headers is not support yet */
334 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
335 return false;
336 }
337 return true;
338 }
339
ch_ipsec_advance_esn_state(struct xfrm_state * x)340 static void ch_ipsec_advance_esn_state(struct xfrm_state *x)
341 {
342 /* do nothing */
343 if (!x->xso.offload_handle)
344 return;
345 }
346
is_eth_imm(const struct sk_buff * skb,struct ipsec_sa_entry * sa_entry)347 static int is_eth_imm(const struct sk_buff *skb,
348 struct ipsec_sa_entry *sa_entry)
349 {
350 unsigned int kctx_len;
351 int hdrlen;
352
353 kctx_len = sa_entry->kctx_len;
354 hdrlen = sizeof(struct fw_ulptx_wr) +
355 sizeof(struct chcr_ipsec_req) + kctx_len;
356
357 hdrlen += sizeof(struct cpl_tx_pkt);
358 if (sa_entry->esn)
359 hdrlen += (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16)
360 << 4);
361 if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
362 return hdrlen;
363 return 0;
364 }
365
calc_tx_sec_flits(const struct sk_buff * skb,struct ipsec_sa_entry * sa_entry,bool * immediate)366 static unsigned int calc_tx_sec_flits(const struct sk_buff *skb,
367 struct ipsec_sa_entry *sa_entry,
368 bool *immediate)
369 {
370 unsigned int kctx_len;
371 unsigned int flits;
372 int aadivlen;
373 int hdrlen;
374
375 kctx_len = sa_entry->kctx_len;
376 hdrlen = is_eth_imm(skb, sa_entry);
377 aadivlen = sa_entry->esn ? DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
378 16) : 0;
379 aadivlen <<= 4;
380
381 /* If the skb is small enough, we can pump it out as a work request
382 * with only immediate data. In that case we just have to have the
383 * TX Packet header plus the skb data in the Work Request.
384 */
385
386 if (hdrlen) {
387 *immediate = true;
388 return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
389 }
390
391 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
392
393 /* Otherwise, we're going to have to construct a Scatter gather list
394 * of the skb body and fragments. We also include the flits necessary
395 * for the TX Packet Work Request and CPL. We always have a firmware
396 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
397 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
398 * message or, if we're doing a Large Send Offload, an LSO CPL message
399 * with an embedded TX Packet Write CPL message.
400 */
401 flits += (sizeof(struct fw_ulptx_wr) +
402 sizeof(struct chcr_ipsec_req) +
403 kctx_len +
404 sizeof(struct cpl_tx_pkt_core) +
405 aadivlen) / sizeof(__be64);
406 return flits;
407 }
408
copy_esn_pktxt(struct sk_buff * skb,struct net_device * dev,void * pos,struct ipsec_sa_entry * sa_entry)409 static void *copy_esn_pktxt(struct sk_buff *skb,
410 struct net_device *dev,
411 void *pos,
412 struct ipsec_sa_entry *sa_entry)
413 {
414 struct chcr_ipsec_aadiv *aadiv;
415 struct ulptx_idata *sc_imm;
416 struct ip_esp_hdr *esphdr;
417 struct xfrm_offload *xo;
418 struct sge_eth_txq *q;
419 struct adapter *adap;
420 struct port_info *pi;
421 __be64 seqno;
422 u32 qidx;
423 u32 seqlo;
424 u8 *iv;
425 int eoq;
426 int len;
427
428 pi = netdev_priv(dev);
429 adap = pi->adapter;
430 qidx = skb->queue_mapping;
431 q = &adap->sge.ethtxq[qidx + pi->first_qset];
432
433 /* end of queue, reset pos to start of queue */
434 eoq = (void *)q->q.stat - pos;
435 if (!eoq)
436 pos = q->q.desc;
437
438 len = DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) << 4;
439 memset(pos, 0, len);
440 aadiv = (struct chcr_ipsec_aadiv *)pos;
441 esphdr = (struct ip_esp_hdr *)skb_transport_header(skb);
442 iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
443 xo = xfrm_offload(skb);
444
445 aadiv->spi = (esphdr->spi);
446 seqlo = ntohl(esphdr->seq_no);
447 seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32));
448 memcpy(aadiv->seq_no, &seqno, 8);
449 iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
450 memcpy(aadiv->iv, iv, 8);
451
452 if (is_eth_imm(skb, sa_entry) && !skb_is_nonlinear(skb)) {
453 sc_imm = (struct ulptx_idata *)(pos +
454 (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
455 sizeof(__be64)) << 3));
456 sc_imm->cmd_more = FILL_CMD_MORE(0);
457 sc_imm->len = cpu_to_be32(skb->len);
458 }
459 pos += len;
460 return pos;
461 }
462
copy_cpltx_pktxt(struct sk_buff * skb,struct net_device * dev,void * pos,struct ipsec_sa_entry * sa_entry)463 static void *copy_cpltx_pktxt(struct sk_buff *skb,
464 struct net_device *dev,
465 void *pos,
466 struct ipsec_sa_entry *sa_entry)
467 {
468 struct cpl_tx_pkt_core *cpl;
469 struct sge_eth_txq *q;
470 struct adapter *adap;
471 struct port_info *pi;
472 u32 ctrl0, qidx;
473 u64 cntrl = 0;
474 int left;
475
476 pi = netdev_priv(dev);
477 adap = pi->adapter;
478 qidx = skb->queue_mapping;
479 q = &adap->sge.ethtxq[qidx + pi->first_qset];
480
481 left = (void *)q->q.stat - pos;
482 if (!left)
483 pos = q->q.desc;
484
485 cpl = (struct cpl_tx_pkt_core *)pos;
486
487 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
488 ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
489 TXPKT_PF_V(adap->pf);
490 if (skb_vlan_tag_present(skb)) {
491 q->vlan_ins++;
492 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
493 }
494
495 cpl->ctrl0 = htonl(ctrl0);
496 cpl->pack = htons(0);
497 cpl->len = htons(skb->len);
498 cpl->ctrl1 = cpu_to_be64(cntrl);
499
500 pos += sizeof(struct cpl_tx_pkt_core);
501 /* Copy ESN info for HW */
502 if (sa_entry->esn)
503 pos = copy_esn_pktxt(skb, dev, pos, sa_entry);
504 return pos;
505 }
506
copy_key_cpltx_pktxt(struct sk_buff * skb,struct net_device * dev,void * pos,struct ipsec_sa_entry * sa_entry)507 static void *copy_key_cpltx_pktxt(struct sk_buff *skb,
508 struct net_device *dev,
509 void *pos,
510 struct ipsec_sa_entry *sa_entry)
511 {
512 struct _key_ctx *key_ctx;
513 int left, eoq, key_len;
514 struct sge_eth_txq *q;
515 struct adapter *adap;
516 struct port_info *pi;
517 unsigned int qidx;
518
519 pi = netdev_priv(dev);
520 adap = pi->adapter;
521 qidx = skb->queue_mapping;
522 q = &adap->sge.ethtxq[qidx + pi->first_qset];
523 key_len = sa_entry->kctx_len;
524
525 /* end of queue, reset pos to start of queue */
526 eoq = (void *)q->q.stat - pos;
527 left = eoq;
528 if (!eoq) {
529 pos = q->q.desc;
530 left = 64 * q->q.size;
531 }
532
533 /* Copy the Key context header */
534 key_ctx = (struct _key_ctx *)pos;
535 key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;
536 memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);
537 pos += sizeof(struct _key_ctx);
538 left -= sizeof(struct _key_ctx);
539
540 if (likely(key_len <= left)) {
541 memcpy(key_ctx->key, sa_entry->key, key_len);
542 pos += key_len;
543 } else {
544 memcpy(pos, sa_entry->key, left);
545 memcpy(q->q.desc, sa_entry->key + left,
546 key_len - left);
547 pos = (u8 *)q->q.desc + (key_len - left);
548 }
549 /* Copy CPL TX PKT XT */
550 pos = copy_cpltx_pktxt(skb, dev, pos, sa_entry);
551
552 return pos;
553 }
554
ch_ipsec_crypto_wreq(struct sk_buff * skb,struct net_device * dev,void * pos,int credits,struct ipsec_sa_entry * sa_entry)555 static void *ch_ipsec_crypto_wreq(struct sk_buff *skb,
556 struct net_device *dev,
557 void *pos,
558 int credits,
559 struct ipsec_sa_entry *sa_entry)
560 {
561 struct port_info *pi = netdev_priv(dev);
562 struct adapter *adap = pi->adapter;
563 unsigned int ivsize = GCM_ESP_IV_SIZE;
564 struct chcr_ipsec_wr *wr;
565 bool immediate = false;
566 u16 immdatalen = 0;
567 unsigned int flits;
568 u32 ivinoffset;
569 u32 aadstart;
570 u32 aadstop;
571 u32 ciphstart;
572 u16 sc_more = 0;
573 u32 ivdrop = 0;
574 u32 esnlen = 0;
575 u32 wr_mid;
576 u16 ndesc;
577 int qidx = skb_get_queue_mapping(skb);
578 struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
579 unsigned int kctx_len = sa_entry->kctx_len;
580 int qid = q->q.cntxt_id;
581
582 atomic_inc(&adap->ch_ipsec_stats.ipsec_cnt);
583
584 flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
585 ndesc = DIV_ROUND_UP(flits, 2);
586 if (sa_entry->esn)
587 ivdrop = 1;
588
589 if (immediate)
590 immdatalen = skb->len;
591
592 if (sa_entry->esn) {
593 esnlen = sizeof(struct chcr_ipsec_aadiv);
594 if (!skb_is_nonlinear(skb))
595 sc_more = 1;
596 }
597
598 /* WR Header */
599 wr = (struct chcr_ipsec_wr *)pos;
600 wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
601 wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(ndesc);
602
603 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
604 netif_tx_stop_queue(q->txq);
605 q->q.stops++;
606 if (!q->dbqt)
607 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
608 }
609 wr_mid |= FW_ULPTX_WR_DATA_F;
610 wr->wreq.flowid_len16 = htonl(wr_mid);
611
612 /* ULPTX */
613 wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
614 wr->req.ulptx.len = htonl(ndesc - 1);
615
616 /* Sub-command */
617 wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen || sc_more);
618 wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
619 sizeof(wr->req.key_ctx) +
620 kctx_len +
621 sizeof(struct cpl_tx_pkt_core) +
622 esnlen +
623 (esnlen ? 0 : immdatalen));
624
625 /* CPL_SEC_PDU */
626 ivinoffset = sa_entry->esn ? (ESN_IV_INSERT_OFFSET + 1) :
627 (skb_transport_offset(skb) +
628 sizeof(struct ip_esp_hdr) + 1);
629 wr->req.sec_cpl.op_ivinsrtofst = htonl(
630 CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
631 CPL_TX_SEC_PDU_CPLLEN_V(2) |
632 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
633 CPL_TX_SEC_PDU_IVINSRTOFST_V(
634 ivinoffset));
635
636 wr->req.sec_cpl.pldlen = htonl(skb->len + esnlen);
637 aadstart = sa_entry->esn ? 1 : (skb_transport_offset(skb) + 1);
638 aadstop = sa_entry->esn ? ESN_IV_INSERT_OFFSET :
639 (skb_transport_offset(skb) +
640 sizeof(struct ip_esp_hdr));
641 ciphstart = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr) +
642 GCM_ESP_IV_SIZE + 1;
643 ciphstart += sa_entry->esn ? esnlen : 0;
644
645 wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
646 aadstart,
647 aadstop,
648 ciphstart, 0);
649
650 wr->req.sec_cpl.cipherstop_lo_authinsert =
651 FILL_SEC_CPL_AUTHINSERT(0, ciphstart,
652 sa_entry->authsize,
653 sa_entry->authsize);
654 wr->req.sec_cpl.seqno_numivs =
655 FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
656 CHCR_SCMD_CIPHER_MODE_AES_GCM,
657 CHCR_SCMD_AUTH_MODE_GHASH,
658 sa_entry->hmac_ctrl,
659 ivsize >> 1);
660 wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
661 0, ivdrop, 0);
662
663 pos += sizeof(struct fw_ulptx_wr) +
664 sizeof(struct ulp_txpkt) +
665 sizeof(struct ulptx_idata) +
666 sizeof(struct cpl_tx_sec_pdu);
667
668 pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
669
670 return pos;
671 }
672
673 /**
674 * flits_to_desc - returns the num of Tx descriptors for the given flits
675 * @n: the number of flits
676 *
677 * Returns the number of Tx descriptors needed for the supplied number
678 * of flits.
679 */
flits_to_desc(unsigned int n)680 static unsigned int flits_to_desc(unsigned int n)
681 {
682 WARN_ON(n > SGE_MAX_WR_LEN / 8);
683 return DIV_ROUND_UP(n, 8);
684 }
685
txq_avail(const struct sge_txq * q)686 static unsigned int txq_avail(const struct sge_txq *q)
687 {
688 return q->size - 1 - q->in_use;
689 }
690
eth_txq_stop(struct sge_eth_txq * q)691 static void eth_txq_stop(struct sge_eth_txq *q)
692 {
693 netif_tx_stop_queue(q->txq);
694 q->q.stops++;
695 }
696
txq_advance(struct sge_txq * q,unsigned int n)697 static void txq_advance(struct sge_txq *q, unsigned int n)
698 {
699 q->in_use += n;
700 q->pidx += n;
701 if (q->pidx >= q->size)
702 q->pidx -= q->size;
703 }
704
705 /*
706 * ch_ipsec_xmit called from ULD Tx handler
707 */
ch_ipsec_xmit(struct sk_buff * skb,struct net_device * dev)708 int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)
709 {
710 struct xfrm_state *x = xfrm_input_state(skb);
711 unsigned int last_desc, ndesc, flits = 0;
712 struct ipsec_sa_entry *sa_entry;
713 u64 *pos, *end, *before, *sgl;
714 struct tx_sw_desc *sgl_sdesc;
715 int qidx, left, credits;
716 bool immediate = false;
717 struct sge_eth_txq *q;
718 struct adapter *adap;
719 struct port_info *pi;
720 struct sec_path *sp;
721
722 if (!x->xso.offload_handle)
723 return NETDEV_TX_BUSY;
724
725 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
726
727 sp = skb_sec_path(skb);
728 if (sp->len != 1) {
729 out_free: dev_kfree_skb_any(skb);
730 return NETDEV_TX_OK;
731 }
732
733 pi = netdev_priv(dev);
734 adap = pi->adapter;
735 qidx = skb->queue_mapping;
736 q = &adap->sge.ethtxq[qidx + pi->first_qset];
737
738 cxgb4_reclaim_completed_tx(adap, &q->q, true);
739
740 flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
741 ndesc = flits_to_desc(flits);
742 credits = txq_avail(&q->q) - ndesc;
743
744 if (unlikely(credits < 0)) {
745 eth_txq_stop(q);
746 dev_err(adap->pdev_dev,
747 "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",
748 dev->name, qidx, credits, ndesc, txq_avail(&q->q),
749 flits);
750 return NETDEV_TX_BUSY;
751 }
752
753 last_desc = q->q.pidx + ndesc - 1;
754 if (last_desc >= q->q.size)
755 last_desc -= q->q.size;
756 sgl_sdesc = &q->q.sdesc[last_desc];
757
758 if (!immediate &&
759 unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
760 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
761 q->mapping_err++;
762 goto out_free;
763 }
764
765 pos = (u64 *)&q->q.desc[q->q.pidx];
766 before = (u64 *)pos;
767 end = (u64 *)pos + flits;
768 /* Setup IPSec CPL */
769 pos = (void *)ch_ipsec_crypto_wreq(skb, dev, (void *)pos,
770 credits, sa_entry);
771 if (before > (u64 *)pos) {
772 left = (u8 *)end - (u8 *)q->q.stat;
773 end = (void *)q->q.desc + left;
774 }
775 if (pos == (u64 *)q->q.stat) {
776 left = (u8 *)end - (u8 *)q->q.stat;
777 end = (void *)q->q.desc + left;
778 pos = (void *)q->q.desc;
779 }
780
781 sgl = (void *)pos;
782 if (immediate) {
783 cxgb4_inline_tx_skb(skb, &q->q, sgl);
784 dev_consume_skb_any(skb);
785 } else {
786 cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,
787 0, sgl_sdesc->addr);
788 skb_orphan(skb);
789 sgl_sdesc->skb = skb;
790 }
791 txq_advance(&q->q, ndesc);
792
793 cxgb4_ring_tx_db(adap, &q->q, ndesc);
794 return NETDEV_TX_OK;
795 }
796
ch_ipsec_init(void)797 static int __init ch_ipsec_init(void)
798 {
799 cxgb4_register_uld(CXGB4_ULD_IPSEC, &ch_ipsec_uld_info);
800
801 return 0;
802 }
803
ch_ipsec_exit(void)804 static void __exit ch_ipsec_exit(void)
805 {
806 struct ipsec_uld_ctx *u_ctx, *tmp;
807 struct adapter *adap;
808
809 mutex_lock(&dev_mutex);
810 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
811 adap = pci_get_drvdata(u_ctx->lldi.pdev);
812 atomic_set(&adap->ch_ipsec_stats.ipsec_cnt, 0);
813 list_del(&u_ctx->entry);
814 kfree(u_ctx);
815 }
816 mutex_unlock(&dev_mutex);
817 cxgb4_unregister_uld(CXGB4_ULD_IPSEC);
818 }
819
820 module_init(ch_ipsec_init);
821 module_exit(ch_ipsec_exit);
822
823 MODULE_DESCRIPTION("Crypto IPSEC for Chelsio Terminator cards.");
824 MODULE_LICENSE("GPL");
825 MODULE_AUTHOR("Chelsio Communications");
826 MODULE_VERSION(CHIPSEC_DRV_VERSION);
827
828