1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * ISHTP client logic
4  *
5  * Copyright (c) 2003-2016, Intel Corporation.
6  */
7 
8 #include <linux/slab.h>
9 #include <linux/sched.h>
10 #include <linux/wait.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <asm/cacheflush.h>
14 #include "hbm.h"
15 #include "client.h"
16 
ishtp_cl_get_tx_free_buffer_size(struct ishtp_cl * cl)17 int ishtp_cl_get_tx_free_buffer_size(struct ishtp_cl *cl)
18 {
19 	unsigned long tx_free_flags;
20 	int size;
21 
22 	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
23 	size = cl->tx_ring_free_size * cl->device->fw_client->props.max_msg_length;
24 	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);
25 
26 	return size;
27 }
28 EXPORT_SYMBOL(ishtp_cl_get_tx_free_buffer_size);
29 
ishtp_cl_get_tx_free_rings(struct ishtp_cl * cl)30 int ishtp_cl_get_tx_free_rings(struct ishtp_cl *cl)
31 {
32 	return cl->tx_ring_free_size;
33 }
34 EXPORT_SYMBOL(ishtp_cl_get_tx_free_rings);
35 
36 /**
37  * ishtp_read_list_flush() - Flush read queue
38  * @cl: ishtp client instance
39  *
40  * Used to remove all entries from read queue for a client
41  */
ishtp_read_list_flush(struct ishtp_cl * cl)42 static void ishtp_read_list_flush(struct ishtp_cl *cl)
43 {
44 	struct ishtp_cl_rb *rb;
45 	struct ishtp_cl_rb *next;
46 	unsigned long	flags;
47 
48 	spin_lock_irqsave(&cl->dev->read_list_spinlock, flags);
49 	list_for_each_entry_safe(rb, next, &cl->dev->read_list.list, list)
50 		if (rb->cl && ishtp_cl_cmp_id(cl, rb->cl)) {
51 			list_del(&rb->list);
52 			spin_lock(&cl->free_list_spinlock);
53 			list_add_tail(&rb->list, &cl->free_rb_list.list);
54 			spin_unlock(&cl->free_list_spinlock);
55 		}
56 	spin_unlock_irqrestore(&cl->dev->read_list_spinlock, flags);
57 }
58 
59 /**
60  * ishtp_cl_flush_queues() - Flush all queues for a client
61  * @cl: ishtp client instance
62  *
63  * Used to remove all queues for a client. This is called when a client device
64  * needs reset due to error, S3 resume or during module removal
65  *
66  * Return: 0 on success else -EINVAL if device is NULL
67  */
ishtp_cl_flush_queues(struct ishtp_cl * cl)68 int ishtp_cl_flush_queues(struct ishtp_cl *cl)
69 {
70 	if (WARN_ON(!cl || !cl->dev))
71 		return -EINVAL;
72 
73 	ishtp_read_list_flush(cl);
74 
75 	return 0;
76 }
77 EXPORT_SYMBOL(ishtp_cl_flush_queues);
78 
79 /**
80  * ishtp_cl_init() - Initialize all fields of a client device
81  * @cl: ishtp client instance
82  * @dev: ishtp device
83  *
84  * Initializes a client device fields: Init spinlocks, init queues etc.
85  * This function is called during new client creation
86  */
ishtp_cl_init(struct ishtp_cl * cl,struct ishtp_device * dev)87 static void ishtp_cl_init(struct ishtp_cl *cl, struct ishtp_device *dev)
88 {
89 	memset(cl, 0, sizeof(struct ishtp_cl));
90 	init_waitqueue_head(&cl->wait_ctrl_res);
91 	spin_lock_init(&cl->free_list_spinlock);
92 	spin_lock_init(&cl->in_process_spinlock);
93 	spin_lock_init(&cl->tx_list_spinlock);
94 	spin_lock_init(&cl->tx_free_list_spinlock);
95 	spin_lock_init(&cl->fc_spinlock);
96 	INIT_LIST_HEAD(&cl->link);
97 	cl->dev = dev;
98 
99 	INIT_LIST_HEAD(&cl->free_rb_list.list);
100 	INIT_LIST_HEAD(&cl->tx_list.list);
101 	INIT_LIST_HEAD(&cl->tx_free_list.list);
102 	INIT_LIST_HEAD(&cl->in_process_list.list);
103 
104 	cl->rx_ring_size = CL_DEF_RX_RING_SIZE;
105 	cl->tx_ring_size = CL_DEF_TX_RING_SIZE;
106 	cl->tx_ring_free_size = cl->tx_ring_size;
107 
108 	/* dma */
109 	cl->last_tx_path = CL_TX_PATH_IPC;
110 	cl->last_dma_acked = 1;
111 	cl->last_dma_addr = NULL;
112 	cl->last_ipc_acked = 1;
113 }
114 
115 /**
116  * ishtp_cl_allocate() - allocates client structure and sets it up.
117  * @cl_device: ishtp client device
118  *
119  * Allocate memory for new client device and call to initialize each field.
120  *
121  * Return: The allocated client instance or NULL on failure
122  */
ishtp_cl_allocate(struct ishtp_cl_device * cl_device)123 struct ishtp_cl *ishtp_cl_allocate(struct ishtp_cl_device *cl_device)
124 {
125 	struct ishtp_cl *cl;
126 
127 	cl = kmalloc(sizeof(struct ishtp_cl), GFP_KERNEL);
128 	if (!cl)
129 		return NULL;
130 
131 	ishtp_cl_init(cl, cl_device->ishtp_dev);
132 	return cl;
133 }
134 EXPORT_SYMBOL(ishtp_cl_allocate);
135 
136 /**
137  * ishtp_cl_free() - Frees a client device
138  * @cl: client device instance
139  *
140  * Frees a client device
141  */
ishtp_cl_free(struct ishtp_cl * cl)142 void	ishtp_cl_free(struct ishtp_cl *cl)
143 {
144 	struct ishtp_device *dev;
145 	unsigned long flags;
146 
147 	if (!cl)
148 		return;
149 
150 	dev = cl->dev;
151 	if (!dev)
152 		return;
153 
154 	spin_lock_irqsave(&dev->cl_list_lock, flags);
155 	ishtp_cl_free_rx_ring(cl);
156 	ishtp_cl_free_tx_ring(cl);
157 	kfree(cl);
158 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
159 }
160 EXPORT_SYMBOL(ishtp_cl_free);
161 
162 /**
163  * ishtp_cl_link() - Reserve a host id and link the client instance
164  * @cl: client device instance
165  *
166  * This allocates a single bit in the hostmap. This function will make sure
167  * that not many client sessions are opened at the same time. Once allocated
168  * the client device instance is added to the ishtp device in the current
169  * client list
170  *
171  * Return: 0 or error code on failure
172  */
ishtp_cl_link(struct ishtp_cl * cl)173 int ishtp_cl_link(struct ishtp_cl *cl)
174 {
175 	struct ishtp_device *dev;
176 	unsigned long flags, flags_cl;
177 	int id, ret = 0;
178 
179 	if (WARN_ON(!cl || !cl->dev))
180 		return -EINVAL;
181 
182 	dev = cl->dev;
183 
184 	spin_lock_irqsave(&dev->device_lock, flags);
185 
186 	if (dev->open_handle_count >= ISHTP_MAX_OPEN_HANDLE_COUNT) {
187 		ret = -EMFILE;
188 		goto unlock_dev;
189 	}
190 
191 	id = find_first_zero_bit(dev->host_clients_map, ISHTP_CLIENTS_MAX);
192 
193 	if (id >= ISHTP_CLIENTS_MAX) {
194 		spin_unlock_irqrestore(&dev->device_lock, flags);
195 		dev_err(&cl->device->dev, "id exceeded %d", ISHTP_CLIENTS_MAX);
196 		return -ENOENT;
197 	}
198 
199 	dev->open_handle_count++;
200 	cl->host_client_id = id;
201 	spin_lock_irqsave(&dev->cl_list_lock, flags_cl);
202 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
203 		ret = -ENODEV;
204 		goto unlock_cl;
205 	}
206 	list_add_tail(&cl->link, &dev->cl_list);
207 	set_bit(id, dev->host_clients_map);
208 	cl->state = ISHTP_CL_INITIALIZING;
209 
210 unlock_cl:
211 	spin_unlock_irqrestore(&dev->cl_list_lock, flags_cl);
212 unlock_dev:
213 	spin_unlock_irqrestore(&dev->device_lock, flags);
214 	return ret;
215 }
216 EXPORT_SYMBOL(ishtp_cl_link);
217 
218 /**
219  * ishtp_cl_unlink() - remove fw_cl from the client device list
220  * @cl: client device instance
221  *
222  * Remove a previously linked device to a ishtp device
223  */
ishtp_cl_unlink(struct ishtp_cl * cl)224 void ishtp_cl_unlink(struct ishtp_cl *cl)
225 {
226 	struct ishtp_device *dev;
227 	struct ishtp_cl *pos;
228 	unsigned long	flags;
229 
230 	/* don't shout on error exit path */
231 	if (!cl || !cl->dev)
232 		return;
233 
234 	dev = cl->dev;
235 
236 	spin_lock_irqsave(&dev->device_lock, flags);
237 	if (dev->open_handle_count > 0) {
238 		clear_bit(cl->host_client_id, dev->host_clients_map);
239 		dev->open_handle_count--;
240 	}
241 	spin_unlock_irqrestore(&dev->device_lock, flags);
242 
243 	/*
244 	 * This checks that 'cl' is actually linked into device's structure,
245 	 * before attempting 'list_del'
246 	 */
247 	spin_lock_irqsave(&dev->cl_list_lock, flags);
248 	list_for_each_entry(pos, &dev->cl_list, link)
249 		if (cl->host_client_id == pos->host_client_id) {
250 			list_del_init(&pos->link);
251 			break;
252 		}
253 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
254 }
255 EXPORT_SYMBOL(ishtp_cl_unlink);
256 
257 /**
258  * ishtp_cl_disconnect() - Send disconnect request to firmware
259  * @cl: client device instance
260  *
261  * Send a disconnect request for a client to firmware.
262  *
263  * Return: 0 if successful disconnect response from the firmware or error
264  * code on failure
265  */
ishtp_cl_disconnect(struct ishtp_cl * cl)266 int ishtp_cl_disconnect(struct ishtp_cl *cl)
267 {
268 	struct ishtp_device *dev;
269 
270 	if (WARN_ON(!cl || !cl->dev))
271 		return -ENODEV;
272 
273 	dev = cl->dev;
274 
275 	dev->print_log(dev, "%s() state %d\n", __func__, cl->state);
276 
277 	if (cl->state != ISHTP_CL_DISCONNECTING) {
278 		dev->print_log(dev, "%s() Disconnect in progress\n", __func__);
279 		return 0;
280 	}
281 
282 	if (ishtp_hbm_cl_disconnect_req(dev, cl)) {
283 		dev->print_log(dev, "%s() Failed to disconnect\n", __func__);
284 		dev_err(&cl->device->dev, "failed to disconnect.\n");
285 		return -ENODEV;
286 	}
287 
288 	wait_event_interruptible_timeout(cl->wait_ctrl_res,
289 			(dev->dev_state != ISHTP_DEV_ENABLED ||
290 			cl->state == ISHTP_CL_DISCONNECTED),
291 			ishtp_secs_to_jiffies(ISHTP_CL_CONNECT_TIMEOUT));
292 
293 	/*
294 	 * If FW reset arrived, this will happen. Don't check cl->,
295 	 * as 'cl' may be freed already
296 	 */
297 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
298 		dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",
299 			       __func__);
300 		return -ENODEV;
301 	}
302 
303 	if (cl->state == ISHTP_CL_DISCONNECTED) {
304 		dev->print_log(dev, "%s() successful\n", __func__);
305 		return 0;
306 	}
307 
308 	return -ENODEV;
309 }
310 EXPORT_SYMBOL(ishtp_cl_disconnect);
311 
312 /**
313  * ishtp_cl_is_other_connecting() - Check other client is connecting
314  * @cl: client device instance
315  *
316  * Checks if other client with the same fw client id is connecting
317  *
318  * Return: true if other client is connected else false
319  */
ishtp_cl_is_other_connecting(struct ishtp_cl * cl)320 static bool ishtp_cl_is_other_connecting(struct ishtp_cl *cl)
321 {
322 	struct ishtp_device *dev;
323 	struct ishtp_cl *pos;
324 	unsigned long	flags;
325 
326 	if (WARN_ON(!cl || !cl->dev))
327 		return false;
328 
329 	dev = cl->dev;
330 	spin_lock_irqsave(&dev->cl_list_lock, flags);
331 	list_for_each_entry(pos, &dev->cl_list, link) {
332 		if ((pos->state == ISHTP_CL_CONNECTING) && (pos != cl) &&
333 				cl->fw_client_id == pos->fw_client_id) {
334 			spin_unlock_irqrestore(&dev->cl_list_lock, flags);
335 			return true;
336 		}
337 	}
338 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
339 
340 	return false;
341 }
342 
343 /**
344  * ishtp_cl_connect_to_fw() - Send connect request to firmware
345  * @cl: client device instance
346  *
347  * Send a connect request to the firmware and wait for firmware response.
348  * If there is successful connection response from the firmware, change
349  * client state to ISHTP_CL_CONNECTED, and bind client to related
350  * firmware client_id.
351  *
352  * Return: 0 for success and error code on failure
353  */
ishtp_cl_connect_to_fw(struct ishtp_cl * cl)354 static int ishtp_cl_connect_to_fw(struct ishtp_cl *cl)
355 {
356 	struct ishtp_device *dev;
357 	int rets;
358 
359 	if (WARN_ON(!cl || !cl->dev))
360 		return -ENODEV;
361 
362 	dev = cl->dev;
363 
364 	if (ishtp_cl_is_other_connecting(cl)) {
365 		dev->print_log(dev, "%s() Busy\n", __func__);
366 		return	-EBUSY;
367 	}
368 
369 	if (ishtp_hbm_cl_connect_req(dev, cl)) {
370 		dev->print_log(dev, "%s() HBM connect req fail\n", __func__);
371 		return -ENODEV;
372 	}
373 
374 	rets = wait_event_interruptible_timeout(cl->wait_ctrl_res,
375 				(dev->dev_state == ISHTP_DEV_ENABLED &&
376 				(cl->state == ISHTP_CL_CONNECTED ||
377 				 cl->state == ISHTP_CL_DISCONNECTED)),
378 				ishtp_secs_to_jiffies(
379 					ISHTP_CL_CONNECT_TIMEOUT));
380 	/*
381 	 * If FW reset arrived, this will happen. Don't check cl->,
382 	 * as 'cl' may be freed already
383 	 */
384 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
385 		dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",
386 			       __func__);
387 		return -EFAULT;
388 	}
389 
390 	if (cl->state != ISHTP_CL_CONNECTED) {
391 		dev->print_log(dev, "%s() state != ISHTP_CL_CONNECTED\n",
392 			       __func__);
393 		return -EFAULT;
394 	}
395 
396 	rets = cl->status;
397 	if (rets) {
398 		dev->print_log(dev, "%s() Invalid status\n", __func__);
399 		return rets;
400 	}
401 
402 	rets = ishtp_cl_device_bind(cl);
403 	if (rets) {
404 		dev->print_log(dev, "%s() Bind error\n", __func__);
405 		ishtp_cl_disconnect(cl);
406 		return rets;
407 	}
408 
409 	return rets;
410 }
411 
412 /**
413  * ishtp_cl_connect() - Build connection with firmware
414  * @cl: client device instance
415  *
416  * Call ishtp_cl_connect_to_fw() to connect and bind to firmware. If successful,
417  * allocate RX and TX ring buffers, and start flow control with firmware to
418  * start communication.
419  *
420  * Return: 0 if there is successful connection to the firmware, allocate
421  * ring buffers.
422  */
ishtp_cl_connect(struct ishtp_cl * cl)423 int ishtp_cl_connect(struct ishtp_cl *cl)
424 {
425 	struct ishtp_device *dev;
426 	int rets;
427 
428 	if (!cl || !cl->dev)
429 		return -ENODEV;
430 
431 	dev = cl->dev;
432 
433 	dev->print_log(dev, "%s() current_state = %d\n", __func__, cl->state);
434 
435 	rets = ishtp_cl_connect_to_fw(cl);
436 	if (rets) {
437 		dev->print_log(dev, "%s() Connect to fw failed\n", __func__);
438 		return rets;
439 	}
440 
441 	rets = ishtp_cl_alloc_rx_ring(cl);
442 	if (rets) {
443 		dev->print_log(dev, "%s() Alloc RX ring failed\n", __func__);
444 		/* if failed allocation, disconnect */
445 		ishtp_cl_disconnect(cl);
446 		return rets;
447 	}
448 
449 	rets = ishtp_cl_alloc_tx_ring(cl);
450 	if (rets) {
451 		dev->print_log(dev, "%s() Alloc TX ring failed\n", __func__);
452 		/* if failed allocation, disconnect */
453 		ishtp_cl_free_rx_ring(cl);
454 		ishtp_cl_disconnect(cl);
455 		return rets;
456 	}
457 
458 	/*
459 	 * Upon successful connection and allocation, start flow-control.
460 	 */
461 	rets = ishtp_cl_read_start(cl);
462 
463 	return rets;
464 }
465 EXPORT_SYMBOL(ishtp_cl_connect);
466 
467 /**
468  * ishtp_cl_establish_connection() - Establish connection with the firmware
469  * @cl: client device instance
470  * @uuid: uuid of the client to search
471  * @tx_size: TX ring buffer size
472  * @rx_size: RX ring buffer size
473  * @reset: true if called for reset connection, otherwise for first connection
474  *
475  * This is a helper function for client driver to build connection with firmware.
476  * If it's first time connecting to the firmware, set reset to false, this
477  * function will link client to bus, find client id and send connect request to
478  * the firmware.
479  *
480  * If it's called for reset handler where client lost connection after
481  * firmware reset, set reset to true, this function will reinit client state and
482  * establish connection again. In this case, this function reuses current client
483  * structure and ring buffers to avoid allocation failure and memory fragments.
484  *
485  * Return: 0 for successful connection with the firmware,
486  * or error code on failure
487  */
ishtp_cl_establish_connection(struct ishtp_cl * cl,const guid_t * uuid,int tx_size,int rx_size,bool reset)488 int ishtp_cl_establish_connection(struct ishtp_cl *cl, const guid_t *uuid,
489 				  int tx_size, int rx_size, bool reset)
490 {
491 	struct ishtp_device *dev;
492 	struct ishtp_fw_client *fw_client;
493 	int rets;
494 
495 	if (!cl || !cl->dev)
496 		return -ENODEV;
497 
498 	dev = cl->dev;
499 
500 	ishtp_set_connection_state(cl, ISHTP_CL_INITIALIZING);
501 
502 	/* reinit ishtp_cl structure if call for reset */
503 	if (reset) {
504 		cl->host_client_id = 0;
505 		cl->fw_client_id = 0;
506 		cl->ishtp_flow_ctrl_creds = 0;
507 		cl->out_flow_ctrl_creds = 0;
508 
509 		cl->last_tx_path = CL_TX_PATH_IPC;
510 		cl->last_dma_acked = 1;
511 		cl->last_dma_addr = NULL;
512 		cl->last_ipc_acked = 1;
513 
514 		cl->sending = 0;
515 		cl->err_send_msg = 0;
516 		cl->err_send_fc = 0;
517 
518 		cl->send_msg_cnt_ipc = 0;
519 		cl->send_msg_cnt_dma = 0;
520 		cl->recv_msg_cnt_ipc = 0;
521 		cl->recv_msg_cnt_dma = 0;
522 		cl->recv_msg_num_frags = 0;
523 		cl->ishtp_flow_ctrl_cnt = 0;
524 		cl->out_flow_ctrl_cnt = 0;
525 	}
526 
527 	/* link to bus */
528 	rets = ishtp_cl_link(cl);
529 	if (rets) {
530 		dev->print_log(dev, "%s() ishtp_cl_link failed\n", __func__);
531 		return rets;
532 	}
533 
534 	/* find firmware client */
535 	fw_client = ishtp_fw_cl_get_client(dev, uuid);
536 	if (!fw_client) {
537 		dev->print_log(dev,
538 			       "%s() ish client uuid not found\n", __func__);
539 		return -ENOENT;
540 	}
541 
542 	ishtp_set_tx_ring_size(cl, tx_size);
543 	ishtp_set_rx_ring_size(cl, rx_size);
544 
545 	ishtp_cl_set_fw_client_id(cl, ishtp_get_fw_client_id(fw_client));
546 
547 	ishtp_set_connection_state(cl, ISHTP_CL_CONNECTING);
548 
549 	/*
550 	 * For reset case, not allocate tx/rx ring buffer which are already
551 	 * done in ishtp_cl_connect() during first connection.
552 	 */
553 	if (reset) {
554 		rets = ishtp_cl_connect_to_fw(cl);
555 		if (!rets)
556 			rets = ishtp_cl_read_start(cl);
557 		else
558 			dev->print_log(dev,
559 				"%s() connect to fw failed\n", __func__);
560 	} else {
561 		rets = ishtp_cl_connect(cl);
562 	}
563 
564 	return rets;
565 }
566 EXPORT_SYMBOL(ishtp_cl_establish_connection);
567 
568 /**
569  * ishtp_cl_destroy_connection() - Disconnect with the firmware
570  * @cl: client device instance
571  * @reset: true if called for firmware reset, false for normal disconnection
572  *
573  * This is a helper function for client driver to disconnect with firmware,
574  * unlink to bus and flush message queue.
575  */
ishtp_cl_destroy_connection(struct ishtp_cl * cl,bool reset)576 void ishtp_cl_destroy_connection(struct ishtp_cl *cl, bool reset)
577 {
578 	if (!cl)
579 		return;
580 
581 	if (reset) {
582 		/*
583 		 * For reset case, connection is already lost during fw reset.
584 		 * Just set state to DISCONNECTED is enough.
585 		 */
586 		ishtp_set_connection_state(cl, ISHTP_CL_DISCONNECTED);
587 	} else {
588 		if (cl->state != ISHTP_CL_DISCONNECTED) {
589 			ishtp_set_connection_state(cl, ISHTP_CL_DISCONNECTING);
590 			ishtp_cl_disconnect(cl);
591 		}
592 	}
593 
594 	ishtp_cl_unlink(cl);
595 	ishtp_cl_flush_queues(cl);
596 }
597 EXPORT_SYMBOL(ishtp_cl_destroy_connection);
598 
599 /**
600  * ishtp_cl_read_start() - Prepare to read client message
601  * @cl: client device instance
602  *
603  * Get a free buffer from pool of free read buffers and add to read buffer
604  * pool to add contents. Send a flow control request to firmware to be able
605  * send next message.
606  *
607  * Return: 0 if successful or error code on failure
608  */
ishtp_cl_read_start(struct ishtp_cl * cl)609 int ishtp_cl_read_start(struct ishtp_cl *cl)
610 {
611 	struct ishtp_device *dev;
612 	struct ishtp_cl_rb *rb;
613 	int rets;
614 	int i;
615 	unsigned long	flags;
616 	unsigned long	dev_flags;
617 
618 	if (WARN_ON(!cl || !cl->dev))
619 		return -ENODEV;
620 
621 	dev = cl->dev;
622 
623 	if (cl->state != ISHTP_CL_CONNECTED)
624 		return -ENODEV;
625 
626 	if (dev->dev_state != ISHTP_DEV_ENABLED)
627 		return -ENODEV;
628 
629 	i = ishtp_fw_cl_by_id(dev, cl->fw_client_id);
630 	if (i < 0) {
631 		dev_err(&cl->device->dev, "no such fw client %d\n",
632 			cl->fw_client_id);
633 		return -ENODEV;
634 	}
635 
636 	/* The current rb is the head of the free rb list */
637 	spin_lock_irqsave(&cl->free_list_spinlock, flags);
638 	if (list_empty(&cl->free_rb_list.list)) {
639 		dev_warn(&cl->device->dev,
640 			 "[ishtp-ish] Rx buffers pool is empty\n");
641 		rets = -ENOMEM;
642 		rb = NULL;
643 		spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
644 		goto out;
645 	}
646 	rb = list_entry(cl->free_rb_list.list.next, struct ishtp_cl_rb, list);
647 	list_del_init(&rb->list);
648 	spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
649 
650 	rb->cl = cl;
651 	rb->buf_idx = 0;
652 
653 	INIT_LIST_HEAD(&rb->list);
654 	rets = 0;
655 
656 	/*
657 	 * This must be BEFORE sending flow control -
658 	 * response in ISR may come too fast...
659 	 */
660 	spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
661 	list_add_tail(&rb->list, &dev->read_list.list);
662 	spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
663 	if (ishtp_hbm_cl_flow_control_req(dev, cl)) {
664 		rets = -ENODEV;
665 		goto out;
666 	}
667 out:
668 	/* if ishtp_hbm_cl_flow_control_req failed, return rb to free list */
669 	if (rets && rb) {
670 		spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
671 		list_del(&rb->list);
672 		spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
673 
674 		spin_lock_irqsave(&cl->free_list_spinlock, flags);
675 		list_add_tail(&rb->list, &cl->free_rb_list.list);
676 		spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
677 	}
678 	return rets;
679 }
680 
681 /**
682  * ishtp_cl_send() - Send a message to firmware
683  * @cl: client device instance
684  * @buf: message buffer
685  * @length: length of message
686  *
687  * If the client is correct state to send message, this function gets a buffer
688  * from tx ring buffers, copy the message data and call to send the message
689  * using ishtp_cl_send_msg()
690  *
691  * Return: 0 if successful or error code on failure
692  */
ishtp_cl_send(struct ishtp_cl * cl,uint8_t * buf,size_t length)693 int ishtp_cl_send(struct ishtp_cl *cl, uint8_t *buf, size_t length)
694 {
695 	struct ishtp_device	*dev;
696 	int	id;
697 	struct ishtp_cl_tx_ring	*cl_msg;
698 	int	have_msg_to_send = 0;
699 	unsigned long	tx_flags, tx_free_flags;
700 
701 	if (WARN_ON(!cl || !cl->dev))
702 		return -ENODEV;
703 
704 	dev = cl->dev;
705 
706 	if (cl->state != ISHTP_CL_CONNECTED) {
707 		++cl->err_send_msg;
708 		return -EPIPE;
709 	}
710 
711 	if (dev->dev_state != ISHTP_DEV_ENABLED) {
712 		++cl->err_send_msg;
713 		return -ENODEV;
714 	}
715 
716 	/* Check if we have fw client device */
717 	id = ishtp_fw_cl_by_id(dev, cl->fw_client_id);
718 	if (id < 0) {
719 		++cl->err_send_msg;
720 		return -ENOENT;
721 	}
722 
723 	if (length > dev->fw_clients[id].props.max_msg_length) {
724 		++cl->err_send_msg;
725 		return -EMSGSIZE;
726 	}
727 
728 	/* No free bufs */
729 	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
730 	if (list_empty(&cl->tx_free_list.list)) {
731 		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
732 			tx_free_flags);
733 		++cl->err_send_msg;
734 		return	-ENOMEM;
735 	}
736 
737 	cl_msg = list_first_entry(&cl->tx_free_list.list,
738 		struct ishtp_cl_tx_ring, list);
739 	if (!cl_msg->send_buf.data) {
740 		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
741 			tx_free_flags);
742 		return	-EIO;
743 		/* Should not happen, as free list is pre-allocated */
744 	}
745 	/*
746 	 * This is safe, as 'length' is already checked for not exceeding
747 	 * max ISHTP message size per client
748 	 */
749 	list_del_init(&cl_msg->list);
750 	--cl->tx_ring_free_size;
751 
752 	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);
753 	memcpy(cl_msg->send_buf.data, buf, length);
754 	cl_msg->send_buf.size = length;
755 	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
756 	have_msg_to_send = !list_empty(&cl->tx_list.list);
757 	list_add_tail(&cl_msg->list, &cl->tx_list.list);
758 	spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
759 
760 	if (!have_msg_to_send && cl->ishtp_flow_ctrl_creds > 0)
761 		ishtp_cl_send_msg(dev, cl);
762 
763 	return	0;
764 }
765 EXPORT_SYMBOL(ishtp_cl_send);
766 
767 /**
768  * ishtp_cl_read_complete() - read complete
769  * @rb: Pointer to client request block
770  *
771  * If the message is completely received call ishtp_cl_bus_rx_event()
772  * to process message
773  */
ishtp_cl_read_complete(struct ishtp_cl_rb * rb)774 static void ishtp_cl_read_complete(struct ishtp_cl_rb *rb)
775 {
776 	unsigned long	flags;
777 	int	schedule_work_flag = 0;
778 	struct ishtp_cl	*cl = rb->cl;
779 
780 	spin_lock_irqsave(&cl->in_process_spinlock, flags);
781 	/*
782 	 * if in-process list is empty, then need to schedule
783 	 * the processing thread
784 	 */
785 	schedule_work_flag = list_empty(&cl->in_process_list.list);
786 	list_add_tail(&rb->list, &cl->in_process_list.list);
787 	spin_unlock_irqrestore(&cl->in_process_spinlock, flags);
788 
789 	if (schedule_work_flag)
790 		ishtp_cl_bus_rx_event(cl->device);
791 }
792 
793 /**
794  * ipc_tx_send() - IPC tx send function
795  * @prm: Pointer to client device instance
796  *
797  * Send message over IPC. Message will be split into fragments
798  * if message size is bigger than IPC FIFO size, and all
799  * fragments will be sent one by one.
800  */
ipc_tx_send(void * prm)801 static void ipc_tx_send(void *prm)
802 {
803 	struct ishtp_cl	*cl = prm;
804 	struct ishtp_cl_tx_ring	*cl_msg;
805 	size_t	rem;
806 	struct ishtp_device	*dev = (cl ? cl->dev : NULL);
807 	struct ishtp_msg_hdr	ishtp_hdr;
808 	unsigned long	tx_flags, tx_free_flags;
809 	unsigned char	*pmsg;
810 
811 	if (!dev)
812 		return;
813 
814 	/*
815 	 * Other conditions if some critical error has
816 	 * occurred before this callback is called
817 	 */
818 	if (dev->dev_state != ISHTP_DEV_ENABLED)
819 		return;
820 
821 	if (cl->state != ISHTP_CL_CONNECTED)
822 		return;
823 
824 	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
825 	if (list_empty(&cl->tx_list.list)) {
826 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
827 		return;
828 	}
829 
830 	if (cl->ishtp_flow_ctrl_creds != 1 && !cl->sending) {
831 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
832 		return;
833 	}
834 
835 	if (!cl->sending) {
836 		--cl->ishtp_flow_ctrl_creds;
837 		cl->last_ipc_acked = 0;
838 		cl->last_tx_path = CL_TX_PATH_IPC;
839 		cl->sending = 1;
840 	}
841 
842 	cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,
843 			    list);
844 	rem = cl_msg->send_buf.size - cl->tx_offs;
845 
846 	while (rem > 0) {
847 		ishtp_hdr.host_addr = cl->host_client_id;
848 		ishtp_hdr.fw_addr = cl->fw_client_id;
849 		ishtp_hdr.reserved = 0;
850 		pmsg = cl_msg->send_buf.data + cl->tx_offs;
851 
852 		if (rem <= dev->mtu) {
853 			/* Last fragment or only one packet */
854 			ishtp_hdr.length = rem;
855 			ishtp_hdr.msg_complete = 1;
856 			/* Submit to IPC queue with no callback */
857 			ishtp_write_message(dev, &ishtp_hdr, pmsg);
858 			cl->tx_offs = 0;
859 			cl->sending = 0;
860 
861 			break;
862 		} else {
863 			/* Send ipc fragment */
864 			ishtp_hdr.length = dev->mtu;
865 			ishtp_hdr.msg_complete = 0;
866 			/* All fregments submitted to IPC queue with no callback */
867 			ishtp_write_message(dev, &ishtp_hdr, pmsg);
868 			cl->tx_offs += dev->mtu;
869 			rem = cl_msg->send_buf.size - cl->tx_offs;
870 		}
871 	}
872 
873 	list_del_init(&cl_msg->list);
874 	spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
875 
876 	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
877 	list_add_tail(&cl_msg->list, &cl->tx_free_list.list);
878 	++cl->tx_ring_free_size;
879 	spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
880 		tx_free_flags);
881 }
882 
883 /**
884  * ishtp_cl_send_msg_ipc() -Send message using IPC
885  * @dev: ISHTP device instance
886  * @cl: Pointer to client device instance
887  *
888  * Send message over IPC not using DMA
889  */
ishtp_cl_send_msg_ipc(struct ishtp_device * dev,struct ishtp_cl * cl)890 static void ishtp_cl_send_msg_ipc(struct ishtp_device *dev,
891 				  struct ishtp_cl *cl)
892 {
893 	/* If last DMA message wasn't acked yet, leave this one in Tx queue */
894 	if (cl->last_tx_path == CL_TX_PATH_DMA && cl->last_dma_acked == 0)
895 		return;
896 
897 	cl->tx_offs = 0;
898 	ipc_tx_send(cl);
899 	++cl->send_msg_cnt_ipc;
900 }
901 
902 /**
903  * ishtp_cl_send_msg_dma() -Send message using DMA
904  * @dev: ISHTP device instance
905  * @cl: Pointer to client device instance
906  *
907  * Send message using DMA
908  */
ishtp_cl_send_msg_dma(struct ishtp_device * dev,struct ishtp_cl * cl)909 static void ishtp_cl_send_msg_dma(struct ishtp_device *dev,
910 	struct ishtp_cl *cl)
911 {
912 	struct ishtp_msg_hdr	hdr;
913 	struct dma_xfer_hbm	dma_xfer;
914 	unsigned char	*msg_addr;
915 	int off;
916 	struct ishtp_cl_tx_ring	*cl_msg;
917 	unsigned long tx_flags, tx_free_flags;
918 
919 	/* If last IPC message wasn't acked yet, leave this one in Tx queue */
920 	if (cl->last_tx_path == CL_TX_PATH_IPC && cl->last_ipc_acked == 0)
921 		return;
922 
923 	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
924 	if (list_empty(&cl->tx_list.list)) {
925 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
926 		return;
927 	}
928 
929 	cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,
930 		list);
931 
932 	msg_addr = ishtp_cl_get_dma_send_buf(dev, cl_msg->send_buf.size);
933 	if (!msg_addr) {
934 		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
935 		if (dev->transfer_path == CL_TX_PATH_DEFAULT)
936 			ishtp_cl_send_msg_ipc(dev, cl);
937 		return;
938 	}
939 
940 	list_del_init(&cl_msg->list);	/* Must be before write */
941 	spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
942 
943 	--cl->ishtp_flow_ctrl_creds;
944 	cl->last_dma_acked = 0;
945 	cl->last_dma_addr = msg_addr;
946 	cl->last_tx_path = CL_TX_PATH_DMA;
947 
948 	/* write msg to dma buf */
949 	memcpy(msg_addr, cl_msg->send_buf.data, cl_msg->send_buf.size);
950 
951 	/*
952 	 * if current fw don't support cache snooping, driver have to
953 	 * flush the cache manually.
954 	 */
955 	if (dev->ops->dma_no_cache_snooping &&
956 		dev->ops->dma_no_cache_snooping(dev))
957 		clflush_cache_range(msg_addr, cl_msg->send_buf.size);
958 
959 	/* send dma_xfer hbm msg */
960 	off = msg_addr - (unsigned char *)dev->ishtp_host_dma_tx_buf;
961 	ishtp_hbm_hdr(&hdr, sizeof(struct dma_xfer_hbm));
962 	dma_xfer.hbm = DMA_XFER;
963 	dma_xfer.fw_client_id = cl->fw_client_id;
964 	dma_xfer.host_client_id = cl->host_client_id;
965 	dma_xfer.reserved = 0;
966 	dma_xfer.msg_addr = dev->ishtp_host_dma_tx_buf_phys + off;
967 	dma_xfer.msg_length = cl_msg->send_buf.size;
968 	dma_xfer.reserved2 = 0;
969 	ishtp_write_message(dev, &hdr, (unsigned char *)&dma_xfer);
970 	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
971 	list_add_tail(&cl_msg->list, &cl->tx_free_list.list);
972 	++cl->tx_ring_free_size;
973 	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);
974 	++cl->send_msg_cnt_dma;
975 }
976 
977 /**
978  * ishtp_cl_send_msg() -Send message using DMA or IPC
979  * @dev: ISHTP device instance
980  * @cl: Pointer to client device instance
981  *
982  * Send message using DMA or IPC based on transfer_path
983  */
ishtp_cl_send_msg(struct ishtp_device * dev,struct ishtp_cl * cl)984 void ishtp_cl_send_msg(struct ishtp_device *dev, struct ishtp_cl *cl)
985 {
986 	if (dev->transfer_path == CL_TX_PATH_DMA)
987 		ishtp_cl_send_msg_dma(dev, cl);
988 	else
989 		ishtp_cl_send_msg_ipc(dev, cl);
990 }
991 
992 /**
993  * recv_ishtp_cl_msg() -Receive client message
994  * @dev: ISHTP device instance
995  * @ishtp_hdr: Pointer to message header
996  *
997  * Receive and dispatch ISHTP client messages. This function executes in ISR
998  * or work queue context
999  */
recv_ishtp_cl_msg(struct ishtp_device * dev,struct ishtp_msg_hdr * ishtp_hdr)1000 void recv_ishtp_cl_msg(struct ishtp_device *dev,
1001 		       struct ishtp_msg_hdr *ishtp_hdr)
1002 {
1003 	struct ishtp_cl *cl;
1004 	struct ishtp_cl_rb *rb;
1005 	struct ishtp_cl_rb *new_rb;
1006 	unsigned char *buffer = NULL;
1007 	struct ishtp_cl_rb *complete_rb = NULL;
1008 	unsigned long	flags;
1009 
1010 	if (ishtp_hdr->reserved) {
1011 		dev_err(dev->devc, "corrupted message header.\n");
1012 		goto	eoi;
1013 	}
1014 
1015 	if (ishtp_hdr->length > IPC_PAYLOAD_SIZE) {
1016 		dev_err(dev->devc,
1017 			"ISHTP message length in hdr exceeds IPC MTU\n");
1018 		goto	eoi;
1019 	}
1020 
1021 	spin_lock_irqsave(&dev->read_list_spinlock, flags);
1022 	list_for_each_entry(rb, &dev->read_list.list, list) {
1023 		cl = rb->cl;
1024 		if (!cl || !(cl->host_client_id == ishtp_hdr->host_addr &&
1025 				cl->fw_client_id == ishtp_hdr->fw_addr) ||
1026 				!(cl->state == ISHTP_CL_CONNECTED))
1027 			continue;
1028 
1029 		 /* If no Rx buffer is allocated, disband the rb */
1030 		if (rb->buffer.size == 0 || rb->buffer.data == NULL) {
1031 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
1032 			dev_err(&cl->device->dev,
1033 				"Rx buffer is not allocated.\n");
1034 			list_del(&rb->list);
1035 			ishtp_io_rb_free(rb);
1036 			cl->status = -ENOMEM;
1037 			goto	eoi;
1038 		}
1039 
1040 		/*
1041 		 * If message buffer overflown (exceeds max. client msg
1042 		 * size, drop message and return to free buffer.
1043 		 * Do we need to disconnect such a client? (We don't send
1044 		 * back FC, so communication will be stuck anyway)
1045 		 */
1046 		if (rb->buffer.size < ishtp_hdr->length + rb->buf_idx) {
1047 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
1048 			dev_err(&cl->device->dev,
1049 				"message overflow. size %d len %d idx %ld\n",
1050 				rb->buffer.size, ishtp_hdr->length,
1051 				rb->buf_idx);
1052 			list_del(&rb->list);
1053 			ishtp_cl_io_rb_recycle(rb);
1054 			cl->status = -EIO;
1055 			goto	eoi;
1056 		}
1057 
1058 		buffer = rb->buffer.data + rb->buf_idx;
1059 		dev->ops->ishtp_read(dev, buffer, ishtp_hdr->length);
1060 
1061 		rb->buf_idx += ishtp_hdr->length;
1062 		if (ishtp_hdr->msg_complete) {
1063 			/* Last fragment in message - it's complete */
1064 			cl->status = 0;
1065 			list_del(&rb->list);
1066 			complete_rb = rb;
1067 
1068 			--cl->out_flow_ctrl_creds;
1069 			/*
1070 			 * the whole msg arrived, send a new FC, and add a new
1071 			 * rb buffer for the next coming msg
1072 			 */
1073 			spin_lock(&cl->free_list_spinlock);
1074 
1075 			if (!list_empty(&cl->free_rb_list.list)) {
1076 				new_rb = list_entry(cl->free_rb_list.list.next,
1077 					struct ishtp_cl_rb, list);
1078 				list_del_init(&new_rb->list);
1079 				spin_unlock(&cl->free_list_spinlock);
1080 				new_rb->cl = cl;
1081 				new_rb->buf_idx = 0;
1082 				INIT_LIST_HEAD(&new_rb->list);
1083 				list_add_tail(&new_rb->list,
1084 					&dev->read_list.list);
1085 
1086 				ishtp_hbm_cl_flow_control_req(dev, cl);
1087 			} else {
1088 				spin_unlock(&cl->free_list_spinlock);
1089 			}
1090 		}
1091 		/* One more fragment in message (even if this was last) */
1092 		++cl->recv_msg_num_frags;
1093 
1094 		/*
1095 		 * We can safely break here (and in BH too),
1096 		 * a single input message can go only to a single request!
1097 		 */
1098 		break;
1099 	}
1100 
1101 	spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
1102 	/* If it's nobody's message, just read and discard it */
1103 	if (!buffer) {
1104 		uint8_t	rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE];
1105 
1106 		dev_err(dev->devc, "Dropped Rx msg - no request\n");
1107 		dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length);
1108 		goto	eoi;
1109 	}
1110 
1111 	if (complete_rb) {
1112 		cl = complete_rb->cl;
1113 		cl->ts_rx = ktime_get();
1114 		++cl->recv_msg_cnt_ipc;
1115 		ishtp_cl_read_complete(complete_rb);
1116 	}
1117 eoi:
1118 	return;
1119 }
1120 
1121 /**
1122  * recv_ishtp_cl_msg_dma() -Receive client message
1123  * @dev: ISHTP device instance
1124  * @msg: message pointer
1125  * @hbm: hbm buffer
1126  *
1127  * Receive and dispatch ISHTP client messages using DMA. This function executes
1128  * in ISR or work queue context
1129  */
recv_ishtp_cl_msg_dma(struct ishtp_device * dev,void * msg,struct dma_xfer_hbm * hbm)1130 void recv_ishtp_cl_msg_dma(struct ishtp_device *dev, void *msg,
1131 			   struct dma_xfer_hbm *hbm)
1132 {
1133 	struct ishtp_cl *cl;
1134 	struct ishtp_cl_rb *rb;
1135 	struct ishtp_cl_rb *new_rb;
1136 	unsigned char *buffer = NULL;
1137 	struct ishtp_cl_rb *complete_rb = NULL;
1138 	unsigned long	flags;
1139 
1140 	spin_lock_irqsave(&dev->read_list_spinlock, flags);
1141 
1142 	list_for_each_entry(rb, &dev->read_list.list, list) {
1143 		cl = rb->cl;
1144 		if (!cl || !(cl->host_client_id == hbm->host_client_id &&
1145 				cl->fw_client_id == hbm->fw_client_id) ||
1146 				!(cl->state == ISHTP_CL_CONNECTED))
1147 			continue;
1148 
1149 		/*
1150 		 * If no Rx buffer is allocated, disband the rb
1151 		 */
1152 		if (rb->buffer.size == 0 || rb->buffer.data == NULL) {
1153 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
1154 			dev_err(&cl->device->dev,
1155 				"response buffer is not allocated.\n");
1156 			list_del(&rb->list);
1157 			ishtp_io_rb_free(rb);
1158 			cl->status = -ENOMEM;
1159 			goto	eoi;
1160 		}
1161 
1162 		/*
1163 		 * If message buffer overflown (exceeds max. client msg
1164 		 * size, drop message and return to free buffer.
1165 		 * Do we need to disconnect such a client? (We don't send
1166 		 * back FC, so communication will be stuck anyway)
1167 		 */
1168 		if (rb->buffer.size < hbm->msg_length) {
1169 			spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
1170 			dev_err(&cl->device->dev,
1171 				"message overflow. size %d len %d idx %ld\n",
1172 				rb->buffer.size, hbm->msg_length, rb->buf_idx);
1173 			list_del(&rb->list);
1174 			ishtp_cl_io_rb_recycle(rb);
1175 			cl->status = -EIO;
1176 			goto	eoi;
1177 		}
1178 
1179 		buffer = rb->buffer.data;
1180 
1181 		/*
1182 		 * if current fw don't support cache snooping, driver have to
1183 		 * flush the cache manually.
1184 		 */
1185 		if (dev->ops->dma_no_cache_snooping &&
1186 			dev->ops->dma_no_cache_snooping(dev))
1187 			clflush_cache_range(msg, hbm->msg_length);
1188 
1189 		memcpy(buffer, msg, hbm->msg_length);
1190 		rb->buf_idx = hbm->msg_length;
1191 
1192 		/* Last fragment in message - it's complete */
1193 		cl->status = 0;
1194 		list_del(&rb->list);
1195 		complete_rb = rb;
1196 
1197 		--cl->out_flow_ctrl_creds;
1198 		/*
1199 		 * the whole msg arrived, send a new FC, and add a new
1200 		 * rb buffer for the next coming msg
1201 		 */
1202 		spin_lock(&cl->free_list_spinlock);
1203 
1204 		if (!list_empty(&cl->free_rb_list.list)) {
1205 			new_rb = list_entry(cl->free_rb_list.list.next,
1206 				struct ishtp_cl_rb, list);
1207 			list_del_init(&new_rb->list);
1208 			spin_unlock(&cl->free_list_spinlock);
1209 			new_rb->cl = cl;
1210 			new_rb->buf_idx = 0;
1211 			INIT_LIST_HEAD(&new_rb->list);
1212 			list_add_tail(&new_rb->list,
1213 				&dev->read_list.list);
1214 
1215 			ishtp_hbm_cl_flow_control_req(dev, cl);
1216 		} else {
1217 			spin_unlock(&cl->free_list_spinlock);
1218 		}
1219 
1220 		/* One more fragment in message (this is always last) */
1221 		++cl->recv_msg_num_frags;
1222 
1223 		/*
1224 		 * We can safely break here (and in BH too),
1225 		 * a single input message can go only to a single request!
1226 		 */
1227 		break;
1228 	}
1229 
1230 	spin_unlock_irqrestore(&dev->read_list_spinlock, flags);
1231 	/* If it's nobody's message, just read and discard it */
1232 	if (!buffer) {
1233 		dev_err(dev->devc, "Dropped Rx (DMA) msg - no request\n");
1234 		goto	eoi;
1235 	}
1236 
1237 	if (complete_rb) {
1238 		cl = complete_rb->cl;
1239 		cl->ts_rx = ktime_get();
1240 		++cl->recv_msg_cnt_dma;
1241 		ishtp_cl_read_complete(complete_rb);
1242 	}
1243 eoi:
1244 	return;
1245 }
1246 
ishtp_get_client_data(struct ishtp_cl * cl)1247 void *ishtp_get_client_data(struct ishtp_cl *cl)
1248 {
1249 	return cl->client_data;
1250 }
1251 EXPORT_SYMBOL(ishtp_get_client_data);
1252 
ishtp_set_client_data(struct ishtp_cl * cl,void * data)1253 void ishtp_set_client_data(struct ishtp_cl *cl, void *data)
1254 {
1255 	cl->client_data = data;
1256 }
1257 EXPORT_SYMBOL(ishtp_set_client_data);
1258 
ishtp_get_ishtp_device(struct ishtp_cl * cl)1259 struct ishtp_device *ishtp_get_ishtp_device(struct ishtp_cl *cl)
1260 {
1261 	return cl->dev;
1262 }
1263 EXPORT_SYMBOL(ishtp_get_ishtp_device);
1264 
ishtp_set_tx_ring_size(struct ishtp_cl * cl,int size)1265 void ishtp_set_tx_ring_size(struct ishtp_cl *cl, int size)
1266 {
1267 	cl->tx_ring_size = size;
1268 }
1269 EXPORT_SYMBOL(ishtp_set_tx_ring_size);
1270 
ishtp_set_rx_ring_size(struct ishtp_cl * cl,int size)1271 void ishtp_set_rx_ring_size(struct ishtp_cl *cl, int size)
1272 {
1273 	cl->rx_ring_size = size;
1274 }
1275 EXPORT_SYMBOL(ishtp_set_rx_ring_size);
1276 
ishtp_set_connection_state(struct ishtp_cl * cl,int state)1277 void ishtp_set_connection_state(struct ishtp_cl *cl, int state)
1278 {
1279 	cl->state = state;
1280 }
1281 EXPORT_SYMBOL(ishtp_set_connection_state);
1282 
ishtp_cl_set_fw_client_id(struct ishtp_cl * cl,int fw_client_id)1283 void ishtp_cl_set_fw_client_id(struct ishtp_cl *cl, int fw_client_id)
1284 {
1285 	cl->fw_client_id = fw_client_id;
1286 }
1287 EXPORT_SYMBOL(ishtp_cl_set_fw_client_id);
1288