Lines Matching +full:int +full:- +full:clock +full:- +full:stable +full:- +full:broken
1 // SPDX-License-Identifier: GPL-2.0-or-later
16 #include <linux/dma-mapping.h>
30 "PLL reference clock (pci[freq] or ext[freq], default ext)");
199 #define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */
208 #define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */
217 #define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */
227 #define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */
272 #define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */
273 #define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */
274 #define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */
275 #define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */
276 #define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */
277 #define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */
278 #define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */
280 #define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */
281 #define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */
282 #define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */
283 #define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */
284 #define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */
285 #define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */
286 #define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */
287 #define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */
288 #define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */
348 #define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1)
387 volatile int cmdi, srci, dsti, resi;
388 volatile int cmdu, srcu, dstu, resu;
389 int cmdk, srck, dstk, resk;
403 int irq;
418 int active, started;
431 unsigned int pk_clk_freq;
434 unsigned int rng_wait_time;
594 int num;
600 unsigned int keysize;
605 unsigned int ivsize;
614 return readl(dev->bar[0] + reg); in hifn_read_0()
619 return readl(dev->bar[1] + reg); in hifn_read_1()
624 writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); in hifn_write_0()
629 writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); in hifn_write_1()
634 int i; in hifn_wait_puc()
637 for (i = 10000; i > 0; --i) { in hifn_wait_puc()
646 dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n"); in hifn_wait_puc()
664 static void hifn_reset_dma(struct hifn_device *dev, int full) in hifn_reset_dma()
695 int i; in hifn_next_signature()
733 static int hifn_rng_data_present(struct hwrng *rng, int wait) in hifn_rng_data_present()
735 struct hifn_device *dev = (struct hifn_device *)rng->priv; in hifn_rng_data_present()
738 nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime)); in hifn_rng_data_present()
739 nsec -= dev->rng_wait_time; in hifn_rng_data_present()
748 static int hifn_rng_data_read(struct hwrng *rng, u32 *data) in hifn_rng_data_read()
750 struct hifn_device *dev = (struct hifn_device *)rng->priv; in hifn_rng_data_read()
753 dev->rngtime = ktime_get(); in hifn_rng_data_read()
757 static int hifn_register_rng(struct hifn_device *dev) in hifn_register_rng()
762 dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC, in hifn_register_rng()
763 dev->pk_clk_freq) * 256; in hifn_register_rng()
765 dev->rng.name = dev->name; in hifn_register_rng()
766 dev->rng.data_present = hifn_rng_data_present; in hifn_register_rng()
767 dev->rng.data_read = hifn_rng_data_read; in hifn_register_rng()
768 dev->rng.priv = (unsigned long)dev; in hifn_register_rng()
770 return hwrng_register(&dev->rng); in hifn_register_rng()
775 hwrng_unregister(&dev->rng); in hifn_unregister_rng()
782 static int hifn_init_pubrng(struct hifn_device *dev) in hifn_init_pubrng()
784 int i; in hifn_init_pubrng()
789 for (i = 100; i > 0; --i) { in hifn_init_pubrng()
797 dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n"); in hifn_init_pubrng()
800 dev->dmareg |= HIFN_DMAIER_PUBDONE; in hifn_init_pubrng()
801 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); in hifn_init_pubrng()
803 dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n"); in hifn_init_pubrng()
810 dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n"); in hifn_init_pubrng()
815 dev->rngtime = ktime_get(); in hifn_init_pubrng()
820 static int hifn_enable_crypto(struct hifn_device *dev) in hifn_enable_crypto()
824 int i; in hifn_enable_crypto()
827 if (pci2id[i].pci_vendor == dev->pdev->vendor && in hifn_enable_crypto()
828 pci2id[i].pci_prod == dev->pdev->device) { in hifn_enable_crypto()
835 dev_err(&dev->pdev->dev, "Unknown card!\n"); in hifn_enable_crypto()
836 return -ENODEV; in hifn_enable_crypto()
858 dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev)); in hifn_enable_crypto()
865 struct hifn_dma *dma = dev->desc_virt; in hifn_init_dma()
866 u32 dptr = dev->desc_dma; in hifn_init_dma()
867 int i; in hifn_init_dma()
870 dma->cmdr[i].p = __cpu_to_le32(dptr + in hifn_init_dma()
873 dma->resr[i].p = __cpu_to_le32(dptr + in hifn_init_dma()
877 dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr + in hifn_init_dma()
879 dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr + in hifn_init_dma()
881 dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr + in hifn_init_dma()
883 dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr + in hifn_init_dma()
886 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; in hifn_init_dma()
887 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; in hifn_init_dma()
888 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; in hifn_init_dma()
892 * Initialize the PLL. We need to know the frequency of the reference clock
898 * Unfortunately the PCI clock is not very suitable since the HIFN needs a
899 * stable clock and the PCI clock frequency may vary, so the default is the
900 * external clock. There is no way to find out its frequency, we default to
906 unsigned int freq, m; in hifn_init_pll()
920 …dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\… in hifn_init_pll()
926 pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT; in hifn_init_pll()
932 /* Select clock source and enable clock bypass */ in hifn_init_pll()
936 /* Let the chip lock to the input clock */ in hifn_init_pll()
939 /* Disable clock bypass */ in hifn_init_pll()
953 dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2; in hifn_init_pll()
958 u32 dptr = dev->desc_dma; in hifn_init_registers()
1007 dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | in hifn_init_registers()
1011 dev->dmareg &= ~HIFN_DMAIER_C_WAIT; in hifn_init_registers()
1013 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); in hifn_init_registers()
1032 static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf, in hifn_setup_base_command()
1039 base_cmd->masks = __cpu_to_le16(mask); in hifn_setup_base_command()
1040 base_cmd->total_source_count = in hifn_setup_base_command()
1042 base_cmd->total_dest_count = in hifn_setup_base_command()
1047 base_cmd->session_num = __cpu_to_le16(snum | in hifn_setup_base_command()
1054 static int hifn_setup_crypto_command(struct hifn_device *dev, in hifn_setup_crypto_command()
1056 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode) in hifn_setup_crypto_command()
1058 struct hifn_dma *dma = dev->desc_virt; in hifn_setup_crypto_command()
1065 cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff); in hifn_setup_crypto_command()
1067 cry_cmd->masks = __cpu_to_le16(mode | in hifn_setup_crypto_command()
1070 cry_cmd->header_skip = 0; in hifn_setup_crypto_command()
1071 cry_cmd->reserved = 0; in hifn_setup_crypto_command()
1075 dma->cmdu++; in hifn_setup_crypto_command()
1076 if (dma->cmdu > 1) { in hifn_setup_crypto_command()
1077 dev->dmareg |= HIFN_DMAIER_C_WAIT; in hifn_setup_crypto_command()
1078 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); in hifn_setup_crypto_command()
1090 cmd_len = buf_pos - buf; in hifn_setup_crypto_command()
1095 static int hifn_setup_cmd_desc(struct hifn_device *dev, in hifn_setup_cmd_desc()
1097 void *priv, unsigned int nbytes) in hifn_setup_cmd_desc()
1099 struct hifn_dma *dma = dev->desc_virt; in hifn_setup_cmd_desc()
1100 int cmd_len, sa_idx; in hifn_setup_cmd_desc()
1104 sa_idx = dma->cmdi; in hifn_setup_cmd_desc()
1105 buf_pos = buf = dma->command_bufs[dma->cmdi]; in hifn_setup_cmd_desc()
1108 switch (rctx->op) { in hifn_setup_cmd_desc()
1123 nbytes, mask, dev->snum); in hifn_setup_cmd_desc()
1125 if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { in hifn_setup_cmd_desc()
1128 if (ctx->keysize) in hifn_setup_cmd_desc()
1130 if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) in hifn_setup_cmd_desc()
1133 switch (rctx->mode) { in hifn_setup_cmd_desc()
1150 switch (rctx->type) { in hifn_setup_cmd_desc()
1152 if (ctx->keysize != 16) in hifn_setup_cmd_desc()
1158 if (ctx->keysize != 24) in hifn_setup_cmd_desc()
1164 if (ctx->keysize != 32) in hifn_setup_cmd_desc()
1170 if (ctx->keysize != 24) in hifn_setup_cmd_desc()
1175 if (ctx->keysize != 8) in hifn_setup_cmd_desc()
1184 nbytes, nbytes, ctx->key, ctx->keysize, in hifn_setup_cmd_desc()
1185 rctx->iv, rctx->ivsize, md); in hifn_setup_cmd_desc()
1188 dev->sa[sa_idx] = priv; in hifn_setup_cmd_desc()
1189 dev->started++; in hifn_setup_cmd_desc()
1191 cmd_len = buf_pos - buf; in hifn_setup_cmd_desc()
1192 dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | in hifn_setup_cmd_desc()
1195 if (++dma->cmdi == HIFN_D_CMD_RSIZE) { in hifn_setup_cmd_desc()
1196 dma->cmdr[dma->cmdi].l = __cpu_to_le32( in hifn_setup_cmd_desc()
1199 dma->cmdi = 0; in hifn_setup_cmd_desc()
1201 dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID); in hifn_setup_cmd_desc()
1204 if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) { in hifn_setup_cmd_desc()
1206 dev->flags |= HIFN_FLAG_CMD_BUSY; in hifn_setup_cmd_desc()
1211 return -EINVAL; in hifn_setup_cmd_desc()
1214 static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, in hifn_setup_src_desc()
1215 unsigned int offset, unsigned int size, int last) in hifn_setup_src_desc()
1217 struct hifn_dma *dma = dev->desc_virt; in hifn_setup_src_desc()
1218 int idx; in hifn_setup_src_desc()
1221 addr = dma_map_page(&dev->pdev->dev, page, offset, size, in hifn_setup_src_desc()
1224 idx = dma->srci; in hifn_setup_src_desc()
1226 dma->srcr[idx].p = __cpu_to_le32(addr); in hifn_setup_src_desc()
1227 dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | in hifn_setup_src_desc()
1231 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | in hifn_setup_src_desc()
1237 dma->srci = idx; in hifn_setup_src_desc()
1238 dma->srcu++; in hifn_setup_src_desc()
1240 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { in hifn_setup_src_desc()
1242 dev->flags |= HIFN_FLAG_SRC_BUSY; in hifn_setup_src_desc()
1250 struct hifn_dma *dma = dev->desc_virt; in hifn_setup_res_desc()
1252 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | in hifn_setup_res_desc()
1255 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | in hifn_setup_res_desc()
1259 if (++dma->resi == HIFN_D_RES_RSIZE) { in hifn_setup_res_desc()
1260 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | in hifn_setup_res_desc()
1262 dma->resi = 0; in hifn_setup_res_desc()
1265 dma->resu++; in hifn_setup_res_desc()
1267 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { in hifn_setup_res_desc()
1269 dev->flags |= HIFN_FLAG_RES_BUSY; in hifn_setup_res_desc()
1274 unsigned offset, unsigned size, int last) in hifn_setup_dst_desc()
1276 struct hifn_dma *dma = dev->desc_virt; in hifn_setup_dst_desc()
1277 int idx; in hifn_setup_dst_desc()
1280 addr = dma_map_page(&dev->pdev->dev, page, offset, size, in hifn_setup_dst_desc()
1283 idx = dma->dsti; in hifn_setup_dst_desc()
1284 dma->dstr[idx].p = __cpu_to_le32(addr); in hifn_setup_dst_desc()
1285 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | in hifn_setup_dst_desc()
1289 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | in hifn_setup_dst_desc()
1294 dma->dsti = idx; in hifn_setup_dst_desc()
1295 dma->dstu++; in hifn_setup_dst_desc()
1297 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { in hifn_setup_dst_desc()
1299 dev->flags |= HIFN_FLAG_DST_BUSY; in hifn_setup_dst_desc()
1303 static int hifn_setup_dma(struct hifn_device *dev, in hifn_setup_dma()
1306 unsigned int nbytes, void *priv) in hifn_setup_dma()
1310 unsigned int soff, doff; in hifn_setup_dma()
1311 unsigned int n, len; in hifn_setup_dma()
1316 soff = src->offset; in hifn_setup_dma()
1317 len = min(src->length, n); in hifn_setup_dma()
1319 hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); in hifn_setup_dma()
1322 n -= len; in hifn_setup_dma()
1325 t = &rctx->walk.cache[0]; in hifn_setup_dma()
1328 if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { in hifn_setup_dma()
1332 len = t->length; in hifn_setup_dma()
1336 doff = dst->offset; in hifn_setup_dma()
1337 len = dst->length; in hifn_setup_dma()
1341 hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); in hifn_setup_dma()
1345 n -= len; in hifn_setup_dma()
1353 static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, in hifn_cipher_walk_init()
1354 int num, gfp_t gfp_flags) in hifn_cipher_walk_init()
1356 int i; in hifn_cipher_walk_init()
1359 sg_init_table(w->cache, num); in hifn_cipher_walk_init()
1361 w->num = 0; in hifn_cipher_walk_init()
1369 s = &w->cache[i]; in hifn_cipher_walk_init()
1372 w->num++; in hifn_cipher_walk_init()
1380 int i; in hifn_cipher_walk_exit()
1382 for (i = 0; i < w->num; ++i) { in hifn_cipher_walk_exit()
1383 struct scatterlist *s = &w->cache[i]; in hifn_cipher_walk_exit()
1387 s->length = 0; in hifn_cipher_walk_exit()
1390 w->num = 0; in hifn_cipher_walk_exit()
1393 static int skcipher_add(unsigned int *drestp, struct scatterlist *dst, in skcipher_add()
1394 unsigned int size, unsigned int *nbytesp) in skcipher_add()
1396 unsigned int copy, drest = *drestp, nbytes = *nbytesp; in skcipher_add()
1397 int idx = 0; in skcipher_add()
1400 return -EINVAL; in skcipher_add()
1403 copy = min3(drest, size, dst->length); in skcipher_add()
1405 size -= copy; in skcipher_add()
1406 drest -= copy; in skcipher_add()
1407 nbytes -= copy; in skcipher_add()
1422 static int hifn_cipher_walk(struct skcipher_request *req, in hifn_cipher_walk()
1426 unsigned int nbytes = req->cryptlen, offset, copy, diff; in hifn_cipher_walk()
1427 int idx, tidx, err; in hifn_cipher_walk()
1432 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) in hifn_cipher_walk()
1433 return -EINVAL; in hifn_cipher_walk()
1435 dst = &req->dst[idx]; in hifn_cipher_walk()
1438 __func__, dst->length, dst->offset, offset, nbytes); in hifn_cipher_walk()
1440 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || in hifn_cipher_walk()
1441 !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || in hifn_cipher_walk()
1443 unsigned slen = min(dst->length - offset, nbytes); in hifn_cipher_walk()
1446 t = &w->cache[idx]; in hifn_cipher_walk()
1454 copy = slen & ~(HIFN_D_DST_DALIGN - 1); in hifn_cipher_walk()
1455 diff = slen & (HIFN_D_DST_DALIGN - 1); in hifn_cipher_walk()
1463 * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); in hifn_cipher_walk()
1485 dst = &req->dst[idx]; in hifn_cipher_walk()
1494 t->length = copy; in hifn_cipher_walk()
1495 t->offset = offset; in hifn_cipher_walk()
1497 nbytes -= min(dst->length, nbytes); in hifn_cipher_walk()
1507 static int hifn_setup_session(struct skcipher_request *req) in hifn_setup_session()
1509 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); in hifn_setup_session()
1511 struct hifn_device *dev = ctx->dev; in hifn_setup_session()
1513 unsigned int nbytes = req->cryptlen, idx = 0; in hifn_setup_session()
1514 int err = -EINVAL, sg_num; in hifn_setup_session()
1517 if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) in hifn_setup_session()
1520 rctx->walk.flags = 0; in hifn_setup_session()
1523 dst = &req->dst[idx]; in hifn_setup_session()
1524 dlen = min(dst->length, nbytes); in hifn_setup_session()
1526 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || in hifn_setup_session()
1528 rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; in hifn_setup_session()
1530 nbytes -= dlen; in hifn_setup_session()
1534 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { in hifn_setup_session()
1535 err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); in hifn_setup_session()
1540 sg_num = hifn_cipher_walk(req, &rctx->walk); in hifn_setup_session()
1546 spin_lock_irqsave(&dev->lock, flags); in hifn_setup_session()
1547 if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { in hifn_setup_session()
1548 err = -EAGAIN; in hifn_setup_session()
1552 err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req); in hifn_setup_session()
1556 dev->snum++; in hifn_setup_session()
1558 dev->active = HIFN_DEFAULT_ACTIVE_NUM; in hifn_setup_session()
1559 spin_unlock_irqrestore(&dev->lock, flags); in hifn_setup_session()
1564 spin_unlock_irqrestore(&dev->lock, flags); in hifn_setup_session()
1567 dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, " in hifn_setup_session()
1569 rctx->iv, rctx->ivsize, in hifn_setup_session()
1570 ctx->key, ctx->keysize, in hifn_setup_session()
1571 rctx->mode, rctx->op, rctx->type, err); in hifn_setup_session()
1577 static int hifn_start_device(struct hifn_device *dev) in hifn_start_device()
1579 int err; in hifn_start_device()
1581 dev->started = dev->active = 0; in hifn_start_device()
1599 static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset, in skcipher_get()
1600 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) in skcipher_get()
1602 unsigned int srest = *srestp, nbytes = *nbytesp, copy; in skcipher_get()
1604 int idx = 0; in skcipher_get()
1607 return -EINVAL; in skcipher_get()
1610 copy = min3(srest, dst->length, size); in skcipher_get()
1613 memcpy(daddr + dst->offset + offset, saddr, copy); in skcipher_get()
1616 nbytes -= copy; in skcipher_get()
1617 size -= copy; in skcipher_get()
1618 srest -= copy; in skcipher_get()
1635 static inline void hifn_complete_sa(struct hifn_device *dev, int i) in hifn_complete_sa()
1639 spin_lock_irqsave(&dev->lock, flags); in hifn_complete_sa()
1640 dev->sa[i] = NULL; in hifn_complete_sa()
1641 dev->started--; in hifn_complete_sa()
1642 if (dev->started < 0) in hifn_complete_sa()
1643 dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__, in hifn_complete_sa()
1644 dev->started); in hifn_complete_sa()
1645 spin_unlock_irqrestore(&dev->lock, flags); in hifn_complete_sa()
1646 BUG_ON(dev->started < 0); in hifn_complete_sa()
1649 static void hifn_process_ready(struct skcipher_request *req, int error) in hifn_process_ready()
1653 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { in hifn_process_ready()
1654 unsigned int nbytes = req->cryptlen; in hifn_process_ready()
1655 int idx = 0, err; in hifn_process_ready()
1660 t = &rctx->walk.cache[idx]; in hifn_process_ready()
1661 dst = &req->dst[idx]; in hifn_process_ready()
1663 pr_debug("\n%s: sg_page(t): %p, t->length: %u, " in hifn_process_ready()
1664 "sg_page(dst): %p, dst->length: %u, " in hifn_process_ready()
1666 __func__, sg_page(t), t->length, in hifn_process_ready()
1667 sg_page(dst), dst->length, nbytes); in hifn_process_ready()
1669 if (!t->length) { in hifn_process_ready()
1670 nbytes -= min(dst->length, nbytes); in hifn_process_ready()
1677 err = skcipher_get(saddr, &t->length, t->offset, in hifn_process_ready()
1688 hifn_cipher_walk_exit(&rctx->walk); in hifn_process_ready()
1694 static void hifn_clear_rings(struct hifn_device *dev, int error) in hifn_clear_rings()
1696 struct hifn_dma *dma = dev->desc_virt; in hifn_clear_rings()
1697 int i, u; in hifn_clear_rings()
1699 dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " in hifn_clear_rings()
1701 dma->cmdi, dma->srci, dma->dsti, dma->resi, in hifn_clear_rings()
1702 dma->cmdu, dma->srcu, dma->dstu, dma->resu, in hifn_clear_rings()
1703 dma->cmdk, dma->srck, dma->dstk, dma->resk); in hifn_clear_rings()
1705 i = dma->resk; u = dma->resu; in hifn_clear_rings()
1707 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) in hifn_clear_rings()
1710 if (dev->sa[i]) { in hifn_clear_rings()
1711 dev->success++; in hifn_clear_rings()
1712 dev->reset = 0; in hifn_clear_rings()
1713 hifn_process_ready(dev->sa[i], error); in hifn_clear_rings()
1719 u--; in hifn_clear_rings()
1721 dma->resk = i; dma->resu = u; in hifn_clear_rings()
1723 i = dma->srck; u = dma->srcu; in hifn_clear_rings()
1725 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) in hifn_clear_rings()
1729 u--; in hifn_clear_rings()
1731 dma->srck = i; dma->srcu = u; in hifn_clear_rings()
1733 i = dma->cmdk; u = dma->cmdu; in hifn_clear_rings()
1735 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) in hifn_clear_rings()
1739 u--; in hifn_clear_rings()
1741 dma->cmdk = i; dma->cmdu = u; in hifn_clear_rings()
1743 i = dma->dstk; u = dma->dstu; in hifn_clear_rings()
1745 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) in hifn_clear_rings()
1749 u--; in hifn_clear_rings()
1751 dma->dstk = i; dma->dstu = u; in hifn_clear_rings()
1753 dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " in hifn_clear_rings()
1755 dma->cmdi, dma->srci, dma->dsti, dma->resi, in hifn_clear_rings()
1756 dma->cmdu, dma->srcu, dma->dstu, dma->resu, in hifn_clear_rings()
1757 dma->cmdk, dma->srck, dma->dstk, dma->resk); in hifn_clear_rings()
1765 int reset = 0; in hifn_work()
1768 spin_lock_irqsave(&dev->lock, flags); in hifn_work()
1769 if (dev->active == 0) { in hifn_work()
1770 struct hifn_dma *dma = dev->desc_virt; in hifn_work()
1772 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) { in hifn_work()
1773 dev->flags &= ~HIFN_FLAG_CMD_BUSY; in hifn_work()
1776 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) { in hifn_work()
1777 dev->flags &= ~HIFN_FLAG_SRC_BUSY; in hifn_work()
1780 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) { in hifn_work()
1781 dev->flags &= ~HIFN_FLAG_DST_BUSY; in hifn_work()
1784 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) { in hifn_work()
1785 dev->flags &= ~HIFN_FLAG_RES_BUSY; in hifn_work()
1791 dev->active--; in hifn_work()
1793 if ((dev->prev_success == dev->success) && dev->started) in hifn_work()
1795 dev->prev_success = dev->success; in hifn_work()
1796 spin_unlock_irqrestore(&dev->lock, flags); in hifn_work()
1799 if (++dev->reset >= 5) { in hifn_work()
1800 int i; in hifn_work()
1801 struct hifn_dma *dma = dev->desc_virt; in hifn_work()
1803 dev_info(&dev->pdev->dev, in hifn_work()
1806 r, dev->active, dev->started, in hifn_work()
1807 dev->success, dev->queue.qlen, dev->queue.max_qlen, in hifn_work()
1810 dev_info(&dev->pdev->dev, "%s: res: ", __func__); in hifn_work()
1812 pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]); in hifn_work()
1813 if (dev->sa[i]) { in hifn_work()
1814 hifn_process_ready(dev->sa[i], -ENODEV); in hifn_work()
1823 dev->reset = 0; in hifn_work()
1826 tasklet_schedule(&dev->tasklet); in hifn_work()
1829 schedule_delayed_work(&dev->work, HZ); in hifn_work()
1832 static irqreturn_t hifn_interrupt(int irq, void *data) in hifn_interrupt()
1835 struct hifn_dma *dma = dev->desc_virt; in hifn_interrupt()
1840 dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " in hifn_interrupt()
1842 dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, in hifn_interrupt()
1843 dma->cmdi, dma->srci, dma->dsti, dma->resi, in hifn_interrupt()
1844 dma->cmdu, dma->srcu, dma->dstu, dma->resu); in hifn_interrupt()
1846 if ((dmacsr & dev->dmareg) == 0) in hifn_interrupt()
1849 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg); in hifn_interrupt()
1861 dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", in hifn_interrupt()
1874 dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n", in hifn_interrupt()
1884 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { in hifn_interrupt()
1885 dev_dbg(&dev->pdev->dev, "wait on command.\n"); in hifn_interrupt()
1886 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT); in hifn_interrupt()
1887 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); in hifn_interrupt()
1890 tasklet_schedule(&dev->tasklet); in hifn_interrupt()
1900 struct hifn_dma *dma = dev->desc_virt; in hifn_flush()
1901 int i; in hifn_flush()
1904 struct hifn_desc *d = &dma->resr[i]; in hifn_flush()
1906 if (dev->sa[i]) { in hifn_flush()
1907 hifn_process_ready(dev->sa[i], in hifn_flush()
1908 (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0); in hifn_flush()
1913 spin_lock_irqsave(&dev->lock, flags); in hifn_flush()
1914 while ((async_req = crypto_dequeue_request(&dev->queue))) { in hifn_flush()
1916 spin_unlock_irqrestore(&dev->lock, flags); in hifn_flush()
1918 hifn_process_ready(req, -ENODEV); in hifn_flush()
1920 spin_lock_irqsave(&dev->lock, flags); in hifn_flush()
1922 spin_unlock_irqrestore(&dev->lock, flags); in hifn_flush()
1925 static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key, in hifn_setkey()
1926 unsigned int len) in hifn_setkey()
1929 struct hifn_device *dev = ctx->dev; in hifn_setkey()
1930 int err; in hifn_setkey()
1936 dev->flags &= ~HIFN_FLAG_OLD_KEY; in hifn_setkey()
1938 memcpy(ctx->key, key, len); in hifn_setkey()
1939 ctx->keysize = len; in hifn_setkey()
1944 static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, in hifn_des3_setkey()
1945 unsigned int len) in hifn_des3_setkey()
1948 struct hifn_device *dev = ctx->dev; in hifn_des3_setkey()
1949 int err; in hifn_des3_setkey()
1955 dev->flags &= ~HIFN_FLAG_OLD_KEY; in hifn_des3_setkey()
1957 memcpy(ctx->key, key, len); in hifn_des3_setkey()
1958 ctx->keysize = len; in hifn_des3_setkey()
1963 static int hifn_handle_req(struct skcipher_request *req) in hifn_handle_req()
1965 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); in hifn_handle_req()
1966 struct hifn_device *dev = ctx->dev; in hifn_handle_req()
1967 int err = -EAGAIN; in hifn_handle_req()
1969 if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH) in hifn_handle_req()
1972 if (err == -EAGAIN) { in hifn_handle_req()
1975 spin_lock_irqsave(&dev->lock, flags); in hifn_handle_req()
1976 err = crypto_enqueue_request(&dev->queue, &req->base); in hifn_handle_req()
1977 spin_unlock_irqrestore(&dev->lock, flags); in hifn_handle_req()
1983 static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op, in hifn_setup_crypto_req()
1986 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); in hifn_setup_crypto_req()
1992 if (req->iv && mode != ACRYPTO_MODE_ECB) { in hifn_setup_crypto_req()
2001 if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) { in hifn_setup_crypto_req()
2002 if (ctx->keysize == 24) in hifn_setup_crypto_req()
2004 else if (ctx->keysize == 32) in hifn_setup_crypto_req()
2008 rctx->op = op; in hifn_setup_crypto_req()
2009 rctx->mode = mode; in hifn_setup_crypto_req()
2010 rctx->type = type; in hifn_setup_crypto_req()
2011 rctx->iv = req->iv; in hifn_setup_crypto_req()
2012 rctx->ivsize = ivsize; in hifn_setup_crypto_req()
2023 static int hifn_process_queue(struct hifn_device *dev) in hifn_process_queue()
2028 int err = 0; in hifn_process_queue()
2030 while (dev->started < HIFN_QUEUE_LENGTH) { in hifn_process_queue()
2031 spin_lock_irqsave(&dev->lock, flags); in hifn_process_queue()
2032 backlog = crypto_get_backlog(&dev->queue); in hifn_process_queue()
2033 async_req = crypto_dequeue_request(&dev->queue); in hifn_process_queue()
2034 spin_unlock_irqrestore(&dev->lock, flags); in hifn_process_queue()
2040 crypto_request_complete(backlog, -EINPROGRESS); in hifn_process_queue()
2052 static int hifn_setup_crypto(struct skcipher_request *req, u8 op, in hifn_setup_crypto()
2055 int err; in hifn_setup_crypto()
2056 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); in hifn_setup_crypto()
2057 struct hifn_device *dev = ctx->dev; in hifn_setup_crypto()
2063 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) in hifn_setup_crypto()
2066 return -EINPROGRESS; in hifn_setup_crypto()
2072 static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req) in hifn_encrypt_aes_ecb()
2077 static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req) in hifn_encrypt_aes_cbc()
2086 static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req) in hifn_decrypt_aes_ecb()
2091 static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req) in hifn_decrypt_aes_cbc()
2100 static inline int hifn_encrypt_des_ecb(struct skcipher_request *req) in hifn_encrypt_des_ecb()
2105 static inline int hifn_encrypt_des_cbc(struct skcipher_request *req) in hifn_encrypt_des_cbc()
2114 static inline int hifn_decrypt_des_ecb(struct skcipher_request *req) in hifn_decrypt_des_ecb()
2119 static inline int hifn_decrypt_des_cbc(struct skcipher_request *req) in hifn_decrypt_des_cbc()
2128 static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req) in hifn_encrypt_3des_ecb()
2133 static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req) in hifn_encrypt_3des_cbc()
2140 static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req) in hifn_decrypt_3des_ecb()
2145 static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req) in hifn_decrypt_3des_cbc()
2154 unsigned int bsize;
2163 .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
2174 .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
2188 .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
2199 .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
2213 .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
2223 .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
2235 static int hifn_init_tfm(struct crypto_skcipher *tfm) in hifn_init_tfm()
2241 ctx->dev = ha->dev; in hifn_init_tfm()
2247 static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t) in hifn_alg_alloc()
2250 int err; in hifn_alg_alloc()
2254 return -ENOMEM; in hifn_alg_alloc()
2256 alg->alg = t->skcipher; in hifn_alg_alloc()
2257 alg->alg.init = hifn_init_tfm; in hifn_alg_alloc()
2259 err = -EINVAL; in hifn_alg_alloc()
2260 if (snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, in hifn_alg_alloc()
2261 "%s", t->name) >= CRYPTO_MAX_ALG_NAME) in hifn_alg_alloc()
2263 if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, in hifn_alg_alloc()
2264 "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME) in hifn_alg_alloc()
2267 alg->alg.base.cra_priority = 300; in hifn_alg_alloc()
2268 alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; in hifn_alg_alloc()
2269 alg->alg.base.cra_blocksize = t->bsize; in hifn_alg_alloc()
2270 alg->alg.base.cra_ctxsize = sizeof(struct hifn_context); in hifn_alg_alloc()
2271 alg->alg.base.cra_alignmask = 0; in hifn_alg_alloc()
2272 alg->alg.base.cra_module = THIS_MODULE; in hifn_alg_alloc()
2274 alg->dev = dev; in hifn_alg_alloc()
2276 list_add_tail(&alg->entry, &dev->alg_list); in hifn_alg_alloc()
2278 err = crypto_register_skcipher(&alg->alg); in hifn_alg_alloc()
2280 list_del(&alg->entry); in hifn_alg_alloc()
2292 list_for_each_entry_safe(a, n, &dev->alg_list, entry) { in hifn_unregister_alg()
2293 list_del(&a->entry); in hifn_unregister_alg()
2294 crypto_unregister_skcipher(&a->alg); in hifn_unregister_alg()
2299 static int hifn_register_alg(struct hifn_device *dev) in hifn_register_alg()
2301 int i, err; in hifn_register_alg()
2323 * (like dev->success), but they are used in process in hifn_tasklet_callback()
2324 * context or update is atomic (like setting dev->sa[i] to NULL). in hifn_tasklet_callback()
2328 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) in hifn_tasklet_callback()
2332 static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) in hifn_probe()
2334 int err, i; in hifn_probe()
2343 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); in hifn_probe()
2348 atomic_inc_return(&hifn_dev_number) - 1); in hifn_probe()
2357 dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n"); in hifn_probe()
2358 err = -ENODEV; in hifn_probe()
2365 err = -ENOMEM; in hifn_probe()
2369 INIT_LIST_HEAD(&dev->alg_list); in hifn_probe()
2371 snprintf(dev->name, sizeof(dev->name), "%s", name); in hifn_probe()
2372 spin_lock_init(&dev->lock); in hifn_probe()
2380 dev->bar[i] = ioremap(addr, size); in hifn_probe()
2381 if (!dev->bar[i]) { in hifn_probe()
2382 err = -ENOMEM; in hifn_probe()
2387 dev->desc_virt = dma_alloc_coherent(&pdev->dev, in hifn_probe()
2389 &dev->desc_dma, GFP_KERNEL); in hifn_probe()
2390 if (!dev->desc_virt) { in hifn_probe()
2391 dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); in hifn_probe()
2392 err = -ENOMEM; in hifn_probe()
2396 dev->pdev = pdev; in hifn_probe()
2397 dev->irq = pdev->irq; in hifn_probe()
2400 dev->sa[i] = NULL; in hifn_probe()
2404 tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev); in hifn_probe()
2406 crypto_init_queue(&dev->queue, 1); in hifn_probe()
2408 err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev); in hifn_probe()
2410 dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", in hifn_probe()
2411 dev->irq, err); in hifn_probe()
2412 dev->irq = 0; in hifn_probe()
2428 INIT_DELAYED_WORK(&dev->work, hifn_work); in hifn_probe()
2429 schedule_delayed_work(&dev->work, HZ); in hifn_probe()
2431 dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been " in hifn_probe()
2433 pci_name(pdev), dev->name); in hifn_probe()
2443 free_irq(dev->irq, dev); in hifn_probe()
2444 tasklet_kill(&dev->tasklet); in hifn_probe()
2446 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, in hifn_probe()
2447 dev->desc_dma); in hifn_probe()
2451 if (dev->bar[i]) in hifn_probe()
2452 iounmap(dev->bar[i]); in hifn_probe()
2466 int i; in hifn_remove()
2472 cancel_delayed_work_sync(&dev->work); in hifn_remove()
2479 free_irq(dev->irq, dev); in hifn_remove()
2480 tasklet_kill(&dev->tasklet); in hifn_remove()
2484 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), in hifn_remove()
2485 dev->desc_virt, dev->desc_dma); in hifn_remove()
2487 if (dev->bar[i]) in hifn_remove()
2488 iounmap(dev->bar[i]); in hifn_remove()
2511 static int __init hifn_init(void) in hifn_init()
2513 unsigned int freq; in hifn_init()
2514 int err; in hifn_init()
2518 pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); in hifn_init()
2519 return -EINVAL; in hifn_init()
2523 * For the 7955/7956 the reference clock frequency must be in the in hifn_init()
2524 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz, in hifn_init()
2531 "be in the range of 20-100"); in hifn_init()
2532 return -EINVAL; in hifn_init()
2540 return -ENODEV; in hifn_init()