1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2019-2022 HiSilicon Limited. */
3 
4 #include <linux/bitfield.h>
5 #include <linux/dmaengine.h>
6 #include <linux/init.h>
7 #include <linux/iopoll.h>
8 #include <linux/module.h>
9 #include <linux/pci.h>
10 #include <linux/spinlock.h>
11 #include "virt-dma.h"
12 
13 /* HiSilicon DMA register common field define */
14 #define HISI_DMA_Q_SQ_BASE_L			0x0
15 #define HISI_DMA_Q_SQ_BASE_H			0x4
16 #define HISI_DMA_Q_SQ_DEPTH			0x8
17 #define HISI_DMA_Q_SQ_TAIL_PTR			0xc
18 #define HISI_DMA_Q_CQ_BASE_L			0x10
19 #define HISI_DMA_Q_CQ_BASE_H			0x14
20 #define HISI_DMA_Q_CQ_DEPTH			0x18
21 #define HISI_DMA_Q_CQ_HEAD_PTR			0x1c
22 #define HISI_DMA_Q_CTRL0			0x20
23 #define HISI_DMA_Q_CTRL0_QUEUE_EN		BIT(0)
24 #define HISI_DMA_Q_CTRL0_QUEUE_PAUSE		BIT(4)
25 #define HISI_DMA_Q_CTRL1			0x24
26 #define HISI_DMA_Q_CTRL1_QUEUE_RESET		BIT(0)
27 #define HISI_DMA_Q_FSM_STS			0x30
28 #define HISI_DMA_Q_FSM_STS_MASK			GENMASK(3, 0)
29 #define HISI_DMA_Q_ERR_INT_NUM0			0x84
30 #define HISI_DMA_Q_ERR_INT_NUM1			0x88
31 #define HISI_DMA_Q_ERR_INT_NUM2			0x8c
32 
33 /* HiSilicon IP08 DMA register and field define */
34 #define HISI_DMA_HIP08_MODE			0x217C
35 #define HISI_DMA_HIP08_Q_BASE			0x0
36 #define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN	BIT(2)
37 #define HISI_DMA_HIP08_Q_INT_STS		0x40
38 #define HISI_DMA_HIP08_Q_INT_MSK		0x44
39 #define HISI_DMA_HIP08_Q_INT_STS_MASK		GENMASK(14, 0)
40 #define HISI_DMA_HIP08_Q_ERR_INT_NUM3		0x90
41 #define HISI_DMA_HIP08_Q_ERR_INT_NUM4		0x94
42 #define HISI_DMA_HIP08_Q_ERR_INT_NUM5		0x98
43 #define HISI_DMA_HIP08_Q_ERR_INT_NUM6		0x48
44 #define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT	BIT(24)
45 
46 /* HiSilicon IP09 DMA register and field define */
47 #define HISI_DMA_HIP09_DMA_FLR_DISABLE		0xA00
48 #define HISI_DMA_HIP09_DMA_FLR_DISABLE_B	BIT(0)
49 #define HISI_DMA_HIP09_Q_BASE			0x2000
50 #define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN	GENMASK(31, 28)
51 #define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT		BIT(26)
52 #define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT		BIT(27)
53 #define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE	BIT(2)
54 #define HISI_DMA_HIP09_Q_INT_STS		0x40
55 #define HISI_DMA_HIP09_Q_INT_MSK		0x44
56 #define HISI_DMA_HIP09_Q_INT_STS_MASK		0x1
57 #define HISI_DMA_HIP09_Q_ERR_INT_STS		0x48
58 #define HISI_DMA_HIP09_Q_ERR_INT_MSK		0x4C
59 #define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK	GENMASK(18, 1)
60 #define HISI_DMA_HIP09_PORT_CFG_REG(port_id)	(0x800 + \
61 						(port_id) * 0x20)
62 #define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B	BIT(16)
63 
64 #define HISI_DMA_HIP09_MAX_PORT_NUM		16
65 
66 #define HISI_DMA_HIP08_MSI_NUM			32
67 #define HISI_DMA_HIP08_CHAN_NUM			30
68 #define HISI_DMA_HIP09_MSI_NUM			4
69 #define HISI_DMA_HIP09_CHAN_NUM			4
70 #define HISI_DMA_REVISION_HIP08B		0x21
71 #define HISI_DMA_REVISION_HIP09A		0x30
72 
73 #define HISI_DMA_Q_OFFSET			0x100
74 #define HISI_DMA_Q_DEPTH_VAL			1024
75 
76 #define PCI_BAR_2				2
77 
78 #define HISI_DMA_POLL_Q_STS_DELAY_US		10
79 #define HISI_DMA_POLL_Q_STS_TIME_OUT_US		1000
80 
81 #define HISI_DMA_MAX_DIR_NAME_LEN		128
82 
83 /*
84  * The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they
85  * have the same pci device id but different pci revision.
86  * Unfortunately, they have different register layouts, so two layout
87  * enumerations are defined.
88  */
89 enum hisi_dma_reg_layout {
90 	HISI_DMA_REG_LAYOUT_INVALID = 0,
91 	HISI_DMA_REG_LAYOUT_HIP08,
92 	HISI_DMA_REG_LAYOUT_HIP09
93 };
94 
95 enum hisi_dma_mode {
96 	EP = 0,
97 	RC,
98 };
99 
100 enum hisi_dma_chan_status {
101 	DISABLE = -1,
102 	IDLE = 0,
103 	RUN,
104 	CPL,
105 	PAUSE,
106 	HALT,
107 	ABORT,
108 	WAIT,
109 	BUFFCLR,
110 };
111 
112 struct hisi_dma_sqe {
113 	__le32 dw0;
114 #define OPCODE_MASK			GENMASK(3, 0)
115 #define OPCODE_SMALL_PACKAGE		0x1
116 #define OPCODE_M2M			0x4
117 #define LOCAL_IRQ_EN			BIT(8)
118 #define ATTR_SRC_MASK			GENMASK(14, 12)
119 	__le32 dw1;
120 	__le32 dw2;
121 #define ATTR_DST_MASK			GENMASK(26, 24)
122 	__le32 length;
123 	__le64 src_addr;
124 	__le64 dst_addr;
125 };
126 
127 struct hisi_dma_cqe {
128 	__le32 rsv0;
129 	__le32 rsv1;
130 	__le16 sq_head;
131 	__le16 rsv2;
132 	__le16 rsv3;
133 	__le16 w0;
134 #define STATUS_MASK			GENMASK(15, 1)
135 #define STATUS_SUCC			0x0
136 #define VALID_BIT			BIT(0)
137 };
138 
139 struct hisi_dma_desc {
140 	struct virt_dma_desc vd;
141 	struct hisi_dma_sqe sqe;
142 };
143 
144 struct hisi_dma_chan {
145 	struct virt_dma_chan vc;
146 	struct hisi_dma_dev *hdma_dev;
147 	struct hisi_dma_sqe *sq;
148 	struct hisi_dma_cqe *cq;
149 	dma_addr_t sq_dma;
150 	dma_addr_t cq_dma;
151 	u32 sq_tail;
152 	u32 cq_head;
153 	u32 qp_num;
154 	enum hisi_dma_chan_status status;
155 	struct hisi_dma_desc *desc;
156 };
157 
158 struct hisi_dma_dev {
159 	struct pci_dev *pdev;
160 	void __iomem *base;
161 	struct dma_device dma_dev;
162 	u32 chan_num;
163 	u32 chan_depth;
164 	enum hisi_dma_reg_layout reg_layout;
165 	void __iomem *queue_base; /* queue region start of register */
166 	struct hisi_dma_chan chan[] __counted_by(chan_num);
167 };
168 
169 #ifdef CONFIG_DEBUG_FS
170 
171 static const struct debugfs_reg32 hisi_dma_comm_chan_regs[] = {
172 	{"DMA_QUEUE_SQ_DEPTH                ", 0x0008ull},
173 	{"DMA_QUEUE_SQ_TAIL_PTR             ", 0x000Cull},
174 	{"DMA_QUEUE_CQ_DEPTH                ", 0x0018ull},
175 	{"DMA_QUEUE_CQ_HEAD_PTR             ", 0x001Cull},
176 	{"DMA_QUEUE_CTRL0                   ", 0x0020ull},
177 	{"DMA_QUEUE_CTRL1                   ", 0x0024ull},
178 	{"DMA_QUEUE_FSM_STS                 ", 0x0030ull},
179 	{"DMA_QUEUE_SQ_STS                  ", 0x0034ull},
180 	{"DMA_QUEUE_CQ_TAIL_PTR             ", 0x003Cull},
181 	{"DMA_QUEUE_INT_STS                 ", 0x0040ull},
182 	{"DMA_QUEUE_INT_MSK                 ", 0x0044ull},
183 	{"DMA_QUEUE_INT_RO                  ", 0x006Cull},
184 };
185 
186 static const struct debugfs_reg32 hisi_dma_hip08_chan_regs[] = {
187 	{"DMA_QUEUE_BYTE_CNT                ", 0x0038ull},
188 	{"DMA_ERR_INT_NUM6                  ", 0x0048ull},
189 	{"DMA_QUEUE_DESP0                   ", 0x0050ull},
190 	{"DMA_QUEUE_DESP1                   ", 0x0054ull},
191 	{"DMA_QUEUE_DESP2                   ", 0x0058ull},
192 	{"DMA_QUEUE_DESP3                   ", 0x005Cull},
193 	{"DMA_QUEUE_DESP4                   ", 0x0074ull},
194 	{"DMA_QUEUE_DESP5                   ", 0x0078ull},
195 	{"DMA_QUEUE_DESP6                   ", 0x007Cull},
196 	{"DMA_QUEUE_DESP7                   ", 0x0080ull},
197 	{"DMA_ERR_INT_NUM0                  ", 0x0084ull},
198 	{"DMA_ERR_INT_NUM1                  ", 0x0088ull},
199 	{"DMA_ERR_INT_NUM2                  ", 0x008Cull},
200 	{"DMA_ERR_INT_NUM3                  ", 0x0090ull},
201 	{"DMA_ERR_INT_NUM4                  ", 0x0094ull},
202 	{"DMA_ERR_INT_NUM5                  ", 0x0098ull},
203 	{"DMA_QUEUE_SQ_STS2                 ", 0x00A4ull},
204 };
205 
206 static const struct debugfs_reg32 hisi_dma_hip09_chan_regs[] = {
207 	{"DMA_QUEUE_ERR_INT_STS             ", 0x0048ull},
208 	{"DMA_QUEUE_ERR_INT_MSK             ", 0x004Cull},
209 	{"DFX_SQ_READ_ERR_PTR               ", 0x0068ull},
210 	{"DFX_DMA_ERR_INT_NUM0              ", 0x0084ull},
211 	{"DFX_DMA_ERR_INT_NUM1              ", 0x0088ull},
212 	{"DFX_DMA_ERR_INT_NUM2              ", 0x008Cull},
213 	{"DFX_DMA_QUEUE_SQ_STS2             ", 0x00A4ull},
214 };
215 
216 static const struct debugfs_reg32 hisi_dma_hip08_comm_regs[] = {
217 	{"DMA_ECC_ERR_ADDR                  ", 0x2004ull},
218 	{"DMA_ECC_ECC_CNT                   ", 0x2014ull},
219 	{"COMMON_AND_CH_ERR_STS             ", 0x2030ull},
220 	{"LOCAL_CPL_ID_STS_0                ", 0x20E0ull},
221 	{"LOCAL_CPL_ID_STS_1                ", 0x20E4ull},
222 	{"LOCAL_CPL_ID_STS_2                ", 0x20E8ull},
223 	{"LOCAL_CPL_ID_STS_3                ", 0x20ECull},
224 	{"LOCAL_TLP_NUM                     ", 0x2158ull},
225 	{"SQCQ_TLP_NUM                      ", 0x2164ull},
226 	{"CPL_NUM                           ", 0x2168ull},
227 	{"INF_BACK_PRESS_STS                ", 0x2170ull},
228 	{"DMA_CH_RAS_LEVEL                  ", 0x2184ull},
229 	{"DMA_CM_RAS_LEVEL                  ", 0x2188ull},
230 	{"DMA_CH_ERR_STS                    ", 0x2190ull},
231 	{"DMA_CH_DONE_STS                   ", 0x2194ull},
232 	{"DMA_SQ_TAG_STS_0                  ", 0x21A0ull},
233 	{"DMA_SQ_TAG_STS_1                  ", 0x21A4ull},
234 	{"DMA_SQ_TAG_STS_2                  ", 0x21A8ull},
235 	{"DMA_SQ_TAG_STS_3                  ", 0x21ACull},
236 	{"LOCAL_P_ID_STS_0                  ", 0x21B0ull},
237 	{"LOCAL_P_ID_STS_1                  ", 0x21B4ull},
238 	{"LOCAL_P_ID_STS_2                  ", 0x21B8ull},
239 	{"LOCAL_P_ID_STS_3                  ", 0x21BCull},
240 	{"DMA_PREBUFF_INFO_0                ", 0x2200ull},
241 	{"DMA_CM_TABLE_INFO_0               ", 0x2220ull},
242 	{"DMA_CM_CE_RO                      ", 0x2244ull},
243 	{"DMA_CM_NFE_RO                     ", 0x2248ull},
244 	{"DMA_CM_FE_RO                      ", 0x224Cull},
245 };
246 
247 static const struct debugfs_reg32 hisi_dma_hip09_comm_regs[] = {
248 	{"COMMON_AND_CH_ERR_STS             ", 0x0030ull},
249 	{"DMA_PORT_IDLE_STS                 ", 0x0150ull},
250 	{"DMA_CH_RAS_LEVEL                  ", 0x0184ull},
251 	{"DMA_CM_RAS_LEVEL                  ", 0x0188ull},
252 	{"DMA_CM_CE_RO                      ", 0x0244ull},
253 	{"DMA_CM_NFE_RO                     ", 0x0248ull},
254 	{"DMA_CM_FE_RO                      ", 0x024Cull},
255 	{"DFX_INF_BACK_PRESS_STS0           ", 0x1A40ull},
256 	{"DFX_INF_BACK_PRESS_STS1           ", 0x1A44ull},
257 	{"DFX_INF_BACK_PRESS_STS2           ", 0x1A48ull},
258 	{"DFX_DMA_WRR_DISABLE               ", 0x1A4Cull},
259 	{"DFX_PA_REQ_TLP_NUM                ", 0x1C00ull},
260 	{"DFX_PA_BACK_TLP_NUM               ", 0x1C04ull},
261 	{"DFX_PA_RETRY_TLP_NUM              ", 0x1C08ull},
262 	{"DFX_LOCAL_NP_TLP_NUM              ", 0x1C0Cull},
263 	{"DFX_LOCAL_CPL_HEAD_TLP_NUM        ", 0x1C10ull},
264 	{"DFX_LOCAL_CPL_DATA_TLP_NUM        ", 0x1C14ull},
265 	{"DFX_LOCAL_CPL_EXT_DATA_TLP_NUM    ", 0x1C18ull},
266 	{"DFX_LOCAL_P_HEAD_TLP_NUM          ", 0x1C1Cull},
267 	{"DFX_LOCAL_P_ACK_TLP_NUM           ", 0x1C20ull},
268 	{"DFX_BUF_ALOC_PORT_REQ_NUM         ", 0x1C24ull},
269 	{"DFX_BUF_ALOC_PORT_RESULT_NUM      ", 0x1C28ull},
270 	{"DFX_BUF_FAIL_SIZE_NUM             ", 0x1C2Cull},
271 	{"DFX_BUF_ALOC_SIZE_NUM             ", 0x1C30ull},
272 	{"DFX_BUF_NP_RELEASE_SIZE_NUM       ", 0x1C34ull},
273 	{"DFX_BUF_P_RELEASE_SIZE_NUM        ", 0x1C38ull},
274 	{"DFX_BUF_PORT_RELEASE_SIZE_NUM     ", 0x1C3Cull},
275 	{"DFX_DMA_PREBUF_MEM0_ECC_ERR_ADDR  ", 0x1CA8ull},
276 	{"DFX_DMA_PREBUF_MEM0_ECC_CNT       ", 0x1CACull},
277 	{"DFX_DMA_LOC_NP_OSTB_ECC_ERR_ADDR  ", 0x1CB0ull},
278 	{"DFX_DMA_LOC_NP_OSTB_ECC_CNT       ", 0x1CB4ull},
279 	{"DFX_DMA_PREBUF_MEM1_ECC_ERR_ADDR  ", 0x1CC0ull},
280 	{"DFX_DMA_PREBUF_MEM1_ECC_CNT       ", 0x1CC4ull},
281 	{"DMA_CH_DONE_STS                   ", 0x02E0ull},
282 	{"DMA_CH_ERR_STS                    ", 0x0320ull},
283 };
284 #endif /* CONFIG_DEBUG_FS*/
285 
hisi_dma_get_reg_layout(struct pci_dev * pdev)286 static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev)
287 {
288 	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
289 		return HISI_DMA_REG_LAYOUT_HIP08;
290 	else if (pdev->revision >= HISI_DMA_REVISION_HIP09A)
291 		return HISI_DMA_REG_LAYOUT_HIP09;
292 
293 	return HISI_DMA_REG_LAYOUT_INVALID;
294 }
295 
hisi_dma_get_chan_num(struct pci_dev * pdev)296 static u32 hisi_dma_get_chan_num(struct pci_dev *pdev)
297 {
298 	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
299 		return HISI_DMA_HIP08_CHAN_NUM;
300 
301 	return HISI_DMA_HIP09_CHAN_NUM;
302 }
303 
hisi_dma_get_msi_num(struct pci_dev * pdev)304 static u32 hisi_dma_get_msi_num(struct pci_dev *pdev)
305 {
306 	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
307 		return HISI_DMA_HIP08_MSI_NUM;
308 
309 	return HISI_DMA_HIP09_MSI_NUM;
310 }
311 
hisi_dma_get_queue_base(struct pci_dev * pdev)312 static u32 hisi_dma_get_queue_base(struct pci_dev *pdev)
313 {
314 	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
315 		return HISI_DMA_HIP08_Q_BASE;
316 
317 	return HISI_DMA_HIP09_Q_BASE;
318 }
319 
to_hisi_dma_chan(struct dma_chan * c)320 static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
321 {
322 	return container_of(c, struct hisi_dma_chan, vc.chan);
323 }
324 
to_hisi_dma_desc(struct virt_dma_desc * vd)325 static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
326 {
327 	return container_of(vd, struct hisi_dma_desc, vd);
328 }
329 
hisi_dma_chan_write(void __iomem * base,u32 reg,u32 index,u32 val)330 static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
331 				       u32 val)
332 {
333 	writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET);
334 }
335 
hisi_dma_update_bit(void __iomem * addr,u32 pos,bool val)336 static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
337 {
338 	u32 tmp;
339 
340 	tmp = readl_relaxed(addr);
341 	tmp = val ? tmp | pos : tmp & ~pos;
342 	writel_relaxed(tmp, addr);
343 }
344 
hisi_dma_pause_dma(struct hisi_dma_dev * hdma_dev,u32 index,bool pause)345 static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
346 			       bool pause)
347 {
348 	void __iomem *addr;
349 
350 	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
351 	       index * HISI_DMA_Q_OFFSET;
352 	hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause);
353 }
354 
hisi_dma_enable_dma(struct hisi_dma_dev * hdma_dev,u32 index,bool enable)355 static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
356 				bool enable)
357 {
358 	void __iomem *addr;
359 
360 	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
361 	       index * HISI_DMA_Q_OFFSET;
362 	hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable);
363 }
364 
hisi_dma_mask_irq(struct hisi_dma_dev * hdma_dev,u32 qp_index)365 static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
366 {
367 	void __iomem *q_base = hdma_dev->queue_base;
368 
369 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
370 		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
371 				    qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
372 	else {
373 		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
374 				    qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
375 		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
376 				    qp_index,
377 				    HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
378 	}
379 }
380 
hisi_dma_unmask_irq(struct hisi_dma_dev * hdma_dev,u32 qp_index)381 static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
382 {
383 	void __iomem *q_base = hdma_dev->queue_base;
384 
385 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
386 		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS,
387 				    qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
388 		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
389 				    qp_index, 0);
390 	} else {
391 		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS,
392 				    qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
393 		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS,
394 				    qp_index,
395 				    HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
396 		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
397 				    qp_index, 0);
398 		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
399 				    qp_index, 0);
400 	}
401 }
402 
hisi_dma_do_reset(struct hisi_dma_dev * hdma_dev,u32 index)403 static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
404 {
405 	void __iomem *addr;
406 
407 	addr = hdma_dev->queue_base +
408 	       HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
409 	hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1);
410 }
411 
hisi_dma_reset_qp_point(struct hisi_dma_dev * hdma_dev,u32 index)412 static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
413 {
414 	void __iomem *q_base = hdma_dev->queue_base;
415 
416 	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
417 	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
418 }
419 
hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan * chan,bool disable)420 static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
421 					      bool disable)
422 {
423 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
424 	u32 index = chan->qp_num, tmp;
425 	void __iomem *addr;
426 	int ret;
427 
428 	hisi_dma_pause_dma(hdma_dev, index, true);
429 	hisi_dma_enable_dma(hdma_dev, index, false);
430 	hisi_dma_mask_irq(hdma_dev, index);
431 
432 	addr = hdma_dev->queue_base +
433 	       HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET;
434 
435 	ret = readl_relaxed_poll_timeout(addr, tmp,
436 		FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN,
437 		HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
438 	if (ret) {
439 		dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
440 		WARN_ON(1);
441 	}
442 
443 	hisi_dma_do_reset(hdma_dev, index);
444 	hisi_dma_reset_qp_point(hdma_dev, index);
445 	hisi_dma_pause_dma(hdma_dev, index, false);
446 
447 	if (!disable) {
448 		hisi_dma_enable_dma(hdma_dev, index, true);
449 		hisi_dma_unmask_irq(hdma_dev, index);
450 	}
451 
452 	ret = readl_relaxed_poll_timeout(addr, tmp,
453 		FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE,
454 		HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
455 	if (ret) {
456 		dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
457 		WARN_ON(1);
458 	}
459 }
460 
hisi_dma_free_chan_resources(struct dma_chan * c)461 static void hisi_dma_free_chan_resources(struct dma_chan *c)
462 {
463 	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
464 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
465 
466 	hisi_dma_reset_or_disable_hw_chan(chan, false);
467 	vchan_free_chan_resources(&chan->vc);
468 
469 	memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
470 	memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
471 	chan->sq_tail = 0;
472 	chan->cq_head = 0;
473 	chan->status = DISABLE;
474 }
475 
hisi_dma_desc_free(struct virt_dma_desc * vd)476 static void hisi_dma_desc_free(struct virt_dma_desc *vd)
477 {
478 	kfree(to_hisi_dma_desc(vd));
479 }
480 
481 static struct dma_async_tx_descriptor *
hisi_dma_prep_dma_memcpy(struct dma_chan * c,dma_addr_t dst,dma_addr_t src,size_t len,unsigned long flags)482 hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
483 			 size_t len, unsigned long flags)
484 {
485 	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
486 	struct hisi_dma_desc *desc;
487 
488 	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
489 	if (!desc)
490 		return NULL;
491 
492 	desc->sqe.length = cpu_to_le32(len);
493 	desc->sqe.src_addr = cpu_to_le64(src);
494 	desc->sqe.dst_addr = cpu_to_le64(dst);
495 
496 	return vchan_tx_prep(&chan->vc, &desc->vd, flags);
497 }
498 
499 static enum dma_status
hisi_dma_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)500 hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
501 		   struct dma_tx_state *txstate)
502 {
503 	return dma_cookie_status(c, cookie, txstate);
504 }
505 
hisi_dma_start_transfer(struct hisi_dma_chan * chan)506 static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
507 {
508 	struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
509 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
510 	struct hisi_dma_desc *desc;
511 	struct virt_dma_desc *vd;
512 
513 	vd = vchan_next_desc(&chan->vc);
514 	if (!vd) {
515 		chan->desc = NULL;
516 		return;
517 	}
518 	list_del(&vd->node);
519 	desc = to_hisi_dma_desc(vd);
520 	chan->desc = desc;
521 
522 	memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
523 
524 	/* update other field in sqe */
525 	sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
526 	sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);
527 
528 	/* make sure data has been updated in sqe */
529 	wmb();
530 
531 	/* update sq tail, point to new sqe position */
532 	chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
533 
534 	/* update sq_tail to trigger a new task */
535 	hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR,
536 			    chan->qp_num, chan->sq_tail);
537 }
538 
hisi_dma_issue_pending(struct dma_chan * c)539 static void hisi_dma_issue_pending(struct dma_chan *c)
540 {
541 	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
542 	unsigned long flags;
543 
544 	spin_lock_irqsave(&chan->vc.lock, flags);
545 
546 	if (vchan_issue_pending(&chan->vc) && !chan->desc)
547 		hisi_dma_start_transfer(chan);
548 
549 	spin_unlock_irqrestore(&chan->vc.lock, flags);
550 }
551 
hisi_dma_terminate_all(struct dma_chan * c)552 static int hisi_dma_terminate_all(struct dma_chan *c)
553 {
554 	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
555 	unsigned long flags;
556 	LIST_HEAD(head);
557 
558 	spin_lock_irqsave(&chan->vc.lock, flags);
559 
560 	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
561 	if (chan->desc) {
562 		vchan_terminate_vdesc(&chan->desc->vd);
563 		chan->desc = NULL;
564 	}
565 
566 	vchan_get_all_descriptors(&chan->vc, &head);
567 
568 	spin_unlock_irqrestore(&chan->vc.lock, flags);
569 
570 	vchan_dma_desc_free_list(&chan->vc, &head);
571 	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
572 
573 	return 0;
574 }
575 
hisi_dma_synchronize(struct dma_chan * c)576 static void hisi_dma_synchronize(struct dma_chan *c)
577 {
578 	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
579 
580 	vchan_synchronize(&chan->vc);
581 }
582 
hisi_dma_alloc_qps_mem(struct hisi_dma_dev * hdma_dev)583 static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
584 {
585 	size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
586 	size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
587 	struct device *dev = &hdma_dev->pdev->dev;
588 	struct hisi_dma_chan *chan;
589 	int i;
590 
591 	for (i = 0; i < hdma_dev->chan_num; i++) {
592 		chan = &hdma_dev->chan[i];
593 		chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
594 					       GFP_KERNEL);
595 		if (!chan->sq)
596 			return -ENOMEM;
597 
598 		chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
599 					       GFP_KERNEL);
600 		if (!chan->cq)
601 			return -ENOMEM;
602 	}
603 
604 	return 0;
605 }
606 
hisi_dma_init_hw_qp(struct hisi_dma_dev * hdma_dev,u32 index)607 static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
608 {
609 	struct hisi_dma_chan *chan = &hdma_dev->chan[index];
610 	void __iomem *q_base = hdma_dev->queue_base;
611 	u32 hw_depth = hdma_dev->chan_depth - 1;
612 	void __iomem *addr;
613 	u32 tmp;
614 
615 	/* set sq, cq base */
616 	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index,
617 			    lower_32_bits(chan->sq_dma));
618 	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index,
619 			    upper_32_bits(chan->sq_dma));
620 	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index,
621 			    lower_32_bits(chan->cq_dma));
622 	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index,
623 			    upper_32_bits(chan->cq_dma));
624 
625 	/* set sq, cq depth */
626 	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth);
627 	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth);
628 
629 	/* init sq tail and cq head */
630 	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
631 	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
632 
633 	/* init error interrupt stats */
634 	hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0);
635 	hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0);
636 	hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0);
637 
638 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
639 		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3,
640 				    index, 0);
641 		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4,
642 				    index, 0);
643 		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5,
644 				    index, 0);
645 		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6,
646 				    index, 0);
647 		/*
648 		 * init SQ/CQ direction selecting register.
649 		 * "0" is to local side and "1" is to remote side.
650 		 */
651 		addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
652 		hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0);
653 
654 		/*
655 		 * 0 - Continue to next descriptor if error occurs.
656 		 * 1 - Abort the DMA queue if error occurs.
657 		 */
658 		hisi_dma_update_bit(addr,
659 				    HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0);
660 	} else {
661 		addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
662 
663 		/*
664 		 * init SQ/CQ direction selecting register.
665 		 * "0" is to local side and "1" is to remote side.
666 		 */
667 		hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0);
668 		hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0);
669 
670 		/*
671 		 * 0 - Continue to next descriptor if error occurs.
672 		 * 1 - Abort the DMA queue if error occurs.
673 		 */
674 
675 		tmp = readl_relaxed(addr);
676 		tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN;
677 		writel_relaxed(tmp, addr);
678 
679 		/*
680 		 * 0 - dma should process FLR with CPU.
681 		 * 1 - dma not process FLR, only cpu process FLR.
682 		 */
683 		addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE +
684 		       index * HISI_DMA_Q_OFFSET;
685 		hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0);
686 
687 		addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
688 		hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1);
689 	}
690 }
691 
hisi_dma_enable_qp(struct hisi_dma_dev * hdma_dev,u32 qp_index)692 static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
693 {
694 	hisi_dma_init_hw_qp(hdma_dev, qp_index);
695 	hisi_dma_unmask_irq(hdma_dev, qp_index);
696 	hisi_dma_enable_dma(hdma_dev, qp_index, true);
697 }
698 
hisi_dma_disable_qp(struct hisi_dma_dev * hdma_dev,u32 qp_index)699 static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
700 {
701 	hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true);
702 }
703 
hisi_dma_enable_qps(struct hisi_dma_dev * hdma_dev)704 static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
705 {
706 	int i;
707 
708 	for (i = 0; i < hdma_dev->chan_num; i++) {
709 		hdma_dev->chan[i].qp_num = i;
710 		hdma_dev->chan[i].hdma_dev = hdma_dev;
711 		hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
712 		vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
713 		hisi_dma_enable_qp(hdma_dev, i);
714 	}
715 }
716 
hisi_dma_disable_qps(struct hisi_dma_dev * hdma_dev)717 static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
718 {
719 	int i;
720 
721 	for (i = 0; i < hdma_dev->chan_num; i++) {
722 		hisi_dma_disable_qp(hdma_dev, i);
723 		tasklet_kill(&hdma_dev->chan[i].vc.task);
724 	}
725 }
726 
hisi_dma_irq(int irq,void * data)727 static irqreturn_t hisi_dma_irq(int irq, void *data)
728 {
729 	struct hisi_dma_chan *chan = data;
730 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
731 	struct hisi_dma_desc *desc;
732 	struct hisi_dma_cqe *cqe;
733 	void __iomem *q_base;
734 
735 	spin_lock(&chan->vc.lock);
736 
737 	desc = chan->desc;
738 	cqe = chan->cq + chan->cq_head;
739 	q_base = hdma_dev->queue_base;
740 	if (desc) {
741 		chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
742 		hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR,
743 				    chan->qp_num, chan->cq_head);
744 		if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
745 			vchan_cookie_complete(&desc->vd);
746 			hisi_dma_start_transfer(chan);
747 		} else {
748 			dev_err(&hdma_dev->pdev->dev, "task error!\n");
749 		}
750 	}
751 
752 	spin_unlock(&chan->vc.lock);
753 
754 	return IRQ_HANDLED;
755 }
756 
hisi_dma_request_qps_irq(struct hisi_dma_dev * hdma_dev)757 static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
758 {
759 	struct pci_dev *pdev = hdma_dev->pdev;
760 	int i, ret;
761 
762 	for (i = 0; i < hdma_dev->chan_num; i++) {
763 		ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
764 				       hisi_dma_irq, IRQF_SHARED, "hisi_dma",
765 				       &hdma_dev->chan[i]);
766 		if (ret)
767 			return ret;
768 	}
769 
770 	return 0;
771 }
772 
773 /* This function enables all hw channels in a device */
hisi_dma_enable_hw_channels(struct hisi_dma_dev * hdma_dev)774 static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
775 {
776 	int ret;
777 
778 	ret = hisi_dma_alloc_qps_mem(hdma_dev);
779 	if (ret) {
780 		dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
781 		return ret;
782 	}
783 
784 	ret = hisi_dma_request_qps_irq(hdma_dev);
785 	if (ret) {
786 		dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
787 		return ret;
788 	}
789 
790 	hisi_dma_enable_qps(hdma_dev);
791 
792 	return 0;
793 }
794 
hisi_dma_disable_hw_channels(void * data)795 static void hisi_dma_disable_hw_channels(void *data)
796 {
797 	hisi_dma_disable_qps(data);
798 }
799 
hisi_dma_set_mode(struct hisi_dma_dev * hdma_dev,enum hisi_dma_mode mode)800 static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
801 			      enum hisi_dma_mode mode)
802 {
803 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
804 		writel_relaxed(mode == RC ? 1 : 0,
805 			       hdma_dev->base + HISI_DMA_HIP08_MODE);
806 }
807 
hisi_dma_init_hw(struct hisi_dma_dev * hdma_dev)808 static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev)
809 {
810 	void __iomem *addr;
811 	int i;
812 
813 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {
814 		for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) {
815 			addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i);
816 			hisi_dma_update_bit(addr,
817 				HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1);
818 		}
819 	}
820 }
821 
hisi_dma_init_dma_dev(struct hisi_dma_dev * hdma_dev)822 static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev)
823 {
824 	struct dma_device *dma_dev;
825 
826 	dma_dev = &hdma_dev->dma_dev;
827 	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
828 	dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
829 	dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
830 	dma_dev->device_tx_status = hisi_dma_tx_status;
831 	dma_dev->device_issue_pending = hisi_dma_issue_pending;
832 	dma_dev->device_terminate_all = hisi_dma_terminate_all;
833 	dma_dev->device_synchronize = hisi_dma_synchronize;
834 	dma_dev->directions = BIT(DMA_MEM_TO_MEM);
835 	dma_dev->dev = &hdma_dev->pdev->dev;
836 	INIT_LIST_HEAD(&dma_dev->channels);
837 }
838 
839 /* --- debugfs implementation --- */
840 #ifdef CONFIG_DEBUG_FS
841 #include <linux/debugfs.h>
hisi_dma_get_ch_regs(struct hisi_dma_dev * hdma_dev,u32 * regs_sz)842 static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev,
843 						  u32 *regs_sz)
844 {
845 	struct device *dev = &hdma_dev->pdev->dev;
846 	struct debugfs_reg32 *regs;
847 	u32 regs_sz_comm;
848 
849 	regs_sz_comm = ARRAY_SIZE(hisi_dma_comm_chan_regs);
850 
851 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
852 		*regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip08_chan_regs);
853 	else
854 		*regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip09_chan_regs);
855 
856 	regs = devm_kcalloc(dev, *regs_sz, sizeof(struct debugfs_reg32),
857 			    GFP_KERNEL);
858 	if (!regs)
859 		return NULL;
860 	memcpy(regs, hisi_dma_comm_chan_regs, sizeof(hisi_dma_comm_chan_regs));
861 
862 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
863 		memcpy(regs + regs_sz_comm, hisi_dma_hip08_chan_regs,
864 		       sizeof(hisi_dma_hip08_chan_regs));
865 	else
866 		memcpy(regs + regs_sz_comm, hisi_dma_hip09_chan_regs,
867 		       sizeof(hisi_dma_hip09_chan_regs));
868 
869 	return regs;
870 }
871 
hisi_dma_create_chan_dir(struct hisi_dma_dev * hdma_dev)872 static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev)
873 {
874 	char dir_name[HISI_DMA_MAX_DIR_NAME_LEN];
875 	struct debugfs_regset32 *regsets;
876 	struct debugfs_reg32 *regs;
877 	struct dentry *chan_dir;
878 	struct device *dev;
879 	u32 regs_sz;
880 	int ret;
881 	int i;
882 
883 	dev = &hdma_dev->pdev->dev;
884 
885 	regsets = devm_kcalloc(dev, hdma_dev->chan_num,
886 			       sizeof(*regsets), GFP_KERNEL);
887 	if (!regsets)
888 		return -ENOMEM;
889 
890 	regs = hisi_dma_get_ch_regs(hdma_dev, &regs_sz);
891 	if (!regs)
892 		return -ENOMEM;
893 
894 	for (i = 0; i < hdma_dev->chan_num; i++) {
895 		regsets[i].regs = regs;
896 		regsets[i].nregs = regs_sz;
897 		regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET;
898 		regsets[i].dev = dev;
899 
900 		memset(dir_name, 0, HISI_DMA_MAX_DIR_NAME_LEN);
901 		ret = sprintf(dir_name, "channel%d", i);
902 		if (ret < 0)
903 			return ret;
904 
905 		chan_dir = debugfs_create_dir(dir_name,
906 					      hdma_dev->dma_dev.dbg_dev_root);
907 		debugfs_create_regset32("regs", 0444, chan_dir, &regsets[i]);
908 	}
909 
910 	return 0;
911 }
912 
hisi_dma_create_debugfs(struct hisi_dma_dev * hdma_dev)913 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev)
914 {
915 	struct debugfs_regset32 *regset;
916 	struct device *dev;
917 	int ret;
918 
919 	dev = &hdma_dev->pdev->dev;
920 
921 	if (hdma_dev->dma_dev.dbg_dev_root == NULL)
922 		return;
923 
924 	regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
925 	if (!regset)
926 		return;
927 
928 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
929 		regset->regs = hisi_dma_hip08_comm_regs;
930 		regset->nregs = ARRAY_SIZE(hisi_dma_hip08_comm_regs);
931 	} else {
932 		regset->regs = hisi_dma_hip09_comm_regs;
933 		regset->nregs = ARRAY_SIZE(hisi_dma_hip09_comm_regs);
934 	}
935 	regset->base = hdma_dev->base;
936 	regset->dev = dev;
937 
938 	debugfs_create_regset32("regs", 0444,
939 				hdma_dev->dma_dev.dbg_dev_root, regset);
940 
941 	ret = hisi_dma_create_chan_dir(hdma_dev);
942 	if (ret < 0)
943 		dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n");
944 }
945 #else
hisi_dma_create_debugfs(struct hisi_dma_dev * hdma_dev)946 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { }
947 #endif /* CONFIG_DEBUG_FS*/
948 /* --- debugfs implementation --- */
949 
hisi_dma_probe(struct pci_dev * pdev,const struct pci_device_id * id)950 static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
951 {
952 	enum hisi_dma_reg_layout reg_layout;
953 	struct device *dev = &pdev->dev;
954 	struct hisi_dma_dev *hdma_dev;
955 	struct dma_device *dma_dev;
956 	u32 chan_num;
957 	u32 msi_num;
958 	int ret;
959 
960 	reg_layout = hisi_dma_get_reg_layout(pdev);
961 	if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) {
962 		dev_err(dev, "unsupported device!\n");
963 		return -EINVAL;
964 	}
965 
966 	ret = pcim_enable_device(pdev);
967 	if (ret) {
968 		dev_err(dev, "failed to enable device mem!\n");
969 		return ret;
970 	}
971 
972 	ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
973 	if (ret) {
974 		dev_err(dev, "failed to remap I/O region!\n");
975 		return ret;
976 	}
977 
978 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
979 	if (ret)
980 		return ret;
981 
982 	chan_num = hisi_dma_get_chan_num(pdev);
983 	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
984 				GFP_KERNEL);
985 	if (!hdma_dev)
986 		return -EINVAL;
987 
988 	hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
989 	hdma_dev->pdev = pdev;
990 	hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
991 	hdma_dev->chan_num = chan_num;
992 	hdma_dev->reg_layout = reg_layout;
993 	hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev);
994 
995 	pci_set_drvdata(pdev, hdma_dev);
996 	pci_set_master(pdev);
997 
998 	msi_num = hisi_dma_get_msi_num(pdev);
999 
1000 	/* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */
1001 	ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI);
1002 	if (ret < 0) {
1003 		dev_err(dev, "Failed to allocate MSI vectors!\n");
1004 		return ret;
1005 	}
1006 
1007 	hisi_dma_init_dma_dev(hdma_dev);
1008 
1009 	hisi_dma_set_mode(hdma_dev, RC);
1010 
1011 	hisi_dma_init_hw(hdma_dev);
1012 
1013 	ret = hisi_dma_enable_hw_channels(hdma_dev);
1014 	if (ret < 0) {
1015 		dev_err(dev, "failed to enable hw channel!\n");
1016 		return ret;
1017 	}
1018 
1019 	ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
1020 				       hdma_dev);
1021 	if (ret)
1022 		return ret;
1023 
1024 	dma_dev = &hdma_dev->dma_dev;
1025 	ret = dmaenginem_async_device_register(dma_dev);
1026 	if (ret < 0) {
1027 		dev_err(dev, "failed to register device!\n");
1028 		return ret;
1029 	}
1030 
1031 	hisi_dma_create_debugfs(hdma_dev);
1032 
1033 	return 0;
1034 }
1035 
1036 static const struct pci_device_id hisi_dma_pci_tbl[] = {
1037 	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
1038 	{ 0, }
1039 };
1040 
1041 static struct pci_driver hisi_dma_pci_driver = {
1042 	.name		= "hisi_dma",
1043 	.id_table	= hisi_dma_pci_tbl,
1044 	.probe		= hisi_dma_probe,
1045 };
1046 
1047 module_pci_driver(hisi_dma_pci_driver);
1048 
1049 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
1050 MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
1051 MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
1052 MODULE_LICENSE("GPL v2");
1053 MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
1054