59 #define for_each_handle_rcu(qh, inst)				\  argument
60 	list_for_each_entry_rcu(qh, &inst->handles, list,	\
88 	struct knav_queue *qh;  in knav_queue_notify()  local
94 	for_each_handle_rcu(qh, inst) {  in knav_queue_notify()
95 		if (atomic_read(&qh->notifier_enabled) <= 0)  in knav_queue_notify()
97 		if (WARN_ON(!qh->notifier_fn))  in knav_queue_notify()
99 		this_cpu_inc(qh->stats->notifies);  in knav_queue_notify()
100 		qh->notifier_fn(qh->notifier_fn_arg);  in knav_queue_notify()
220 	struct knav_queue *qh;  in __knav_queue_open()  local
224 	qh = devm_kzalloc(inst->kdev->dev, sizeof(*qh), GFP_KERNEL);  in __knav_queue_open()
225 	if (!qh)  in __knav_queue_open()
228 	qh->stats = alloc_percpu(struct knav_queue_stats);  in __knav_queue_open()
229 	if (!qh->stats) {  in __knav_queue_open()
234 	qh->flags = flags;  in __knav_queue_open()
235 	qh->inst = inst;  in __knav_queue_open()
237 	qh->reg_push = &inst->qmgr->reg_push[id];  in __knav_queue_open()
238 	qh->reg_pop = &inst->qmgr->reg_pop[id];  in __knav_queue_open()
239 	qh->reg_peek = &inst->qmgr->reg_peek[id];  in __knav_queue_open()
252 	list_add_tail_rcu(&qh->list, &inst->handles);  in __knav_queue_open()
253 	return qh;  in __knav_queue_open()
256 	if (qh->stats)  in __knav_queue_open()
257 		free_percpu(qh->stats);  in __knav_queue_open()
258 	devm_kfree(inst->kdev->dev, qh);  in __knav_queue_open()
266 	struct knav_queue *qh;  in knav_queue_open_by_id()  local
270 	qh = ERR_PTR(-ENODEV);  in knav_queue_open_by_id()
275 	qh = ERR_PTR(-EEXIST);  in knav_queue_open_by_id()
279 	qh = ERR_PTR(-EBUSY);  in knav_queue_open_by_id()
284 	qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_id()
289 	return qh;  in knav_queue_open_by_id()
296 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open_by_type()  local
308 		qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_type()
314 	return qh;  in knav_queue_open_by_type()
325 static int knav_queue_enable_notifier(struct knav_queue *qh)  in knav_queue_enable_notifier()  argument
327 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_enable_notifier()
330 	if (WARN_ON(!qh->notifier_fn))  in knav_queue_enable_notifier()
334 	first = (atomic_inc_return(&qh->notifier_enabled) == 1);  in knav_queue_enable_notifier()
346 static int knav_queue_disable_notifier(struct knav_queue *qh)  in knav_queue_disable_notifier()  argument
348 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_disable_notifier()
351 	last = (atomic_dec_return(&qh->notifier_enabled) == 0);  in knav_queue_disable_notifier()
362 static int knav_queue_set_notifier(struct knav_queue *qh,  in knav_queue_set_notifier()  argument
365 	knav_queue_notify_fn old_fn = qh->notifier_fn;  in knav_queue_set_notifier()
370 	if (!(qh->inst->range->flags & (RANGE_HAS_ACCUMULATOR | RANGE_HAS_IRQ)))  in knav_queue_set_notifier()
374 		knav_queue_disable_notifier(qh);  in knav_queue_set_notifier()
376 	qh->notifier_fn = cfg->fn;  in knav_queue_set_notifier()
377 	qh->notifier_fn_arg = cfg->fn_arg;  in knav_queue_set_notifier()
380 		knav_queue_enable_notifier(qh);  in knav_queue_set_notifier()
423 	struct knav_queue *qh = qhandle;  in knav_queue_get_count()  local
424 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_get_count()
426 	return readl_relaxed(&qh->reg_peek[0].entry_count) +  in knav_queue_get_count()
434 	struct knav_queue *qh;  in knav_queue_debug_show_instance()  local
447 	for_each_handle_rcu(qh, inst) {  in knav_queue_debug_show_instance()
449 			pushes += per_cpu_ptr(qh->stats, cpu)->pushes;  in knav_queue_debug_show_instance()
450 			pops += per_cpu_ptr(qh->stats, cpu)->pops;  in knav_queue_debug_show_instance()
451 			push_errors += per_cpu_ptr(qh->stats, cpu)->push_errors;  in knav_queue_debug_show_instance()
452 			pop_errors += per_cpu_ptr(qh->stats, cpu)->pop_errors;  in knav_queue_debug_show_instance()
453 			notifies += per_cpu_ptr(qh->stats, cpu)->notifies;  in knav_queue_debug_show_instance()
457 				qh,  in knav_queue_debug_show_instance()
460 				knav_queue_get_count(qh),  in knav_queue_debug_show_instance()
504 static int knav_queue_flush(struct knav_queue *qh)  in knav_queue_flush()  argument
506 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_flush()
531 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open()  local
537 		qh = knav_queue_open_by_type(name, id, flags);  in knav_queue_open()
541 		qh = knav_queue_open_by_id(name, id, flags);  in knav_queue_open()
544 	return qh;  in knav_queue_open()
554 	struct knav_queue *qh = qhandle;  in knav_queue_close()  local
555 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_close()
557 	while (atomic_read(&qh->notifier_enabled) > 0)  in knav_queue_close()
558 		knav_queue_disable_notifier(qh);  in knav_queue_close()
561 	list_del_rcu(&qh->list);  in knav_queue_close()
570 	free_percpu(qh->stats);  in knav_queue_close()
571 	devm_kfree(inst->kdev->dev, qh);  in knav_queue_close()
586 	struct knav_queue *qh = qhandle;  in knav_queue_device_control()  local
592 		ret = qh->inst->kdev->base_id + qh->inst->id;  in knav_queue_device_control()
596 		ret = knav_queue_flush(qh);  in knav_queue_device_control()
601 		ret = knav_queue_set_notifier(qh, cfg);  in knav_queue_device_control()
605 		ret = knav_queue_enable_notifier(qh);  in knav_queue_device_control()
609 		ret = knav_queue_disable_notifier(qh);  in knav_queue_device_control()
613 		ret = knav_queue_get_count(qh);  in knav_queue_device_control()
638 	struct knav_queue *qh = qhandle;  in knav_queue_push()  local
642 	writel_relaxed(val, &qh->reg_push[0].ptr_size_thresh);  in knav_queue_push()
644 	this_cpu_inc(qh->stats->pushes);  in knav_queue_push()
658 	struct knav_queue *qh = qhandle;  in knav_queue_pop()  local
659 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_pop()
673 		val = readl_relaxed(&qh->reg_pop[0].ptr_size_thresh);  in knav_queue_pop()
682 	this_cpu_inc(qh->stats->pops);  in knav_queue_pop()