219 static u64 fuse_get_unique_locked(struct fuse_iqueue *fiq)  in fuse_get_unique_locked()  argument
221 	fiq->reqctr += FUSE_REQ_ID_STEP;  in fuse_get_unique_locked()
222 	return fiq->reqctr;  in fuse_get_unique_locked()
225 u64 fuse_get_unique(struct fuse_iqueue *fiq)  in fuse_get_unique()  argument
229 	spin_lock(&fiq->lock);  in fuse_get_unique()
230 	ret = fuse_get_unique_locked(fiq);  in fuse_get_unique()
231 	spin_unlock(&fiq->lock);  in fuse_get_unique()
243  * A new request is available, wake fiq->waitq
245 static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)  in fuse_dev_wake_and_unlock()  argument
246 __releases(fiq->lock)  in fuse_dev_wake_and_unlock()
248 	wake_up(&fiq->waitq);  in fuse_dev_wake_and_unlock()
249 	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_dev_wake_and_unlock()
250 	spin_unlock(&fiq->lock);  in fuse_dev_wake_and_unlock()
253 static void fuse_dev_queue_forget(struct fuse_iqueue *fiq, struct fuse_forget_link *forget)  in fuse_dev_queue_forget()  argument
255 	spin_lock(&fiq->lock);  in fuse_dev_queue_forget()
256 	if (fiq->connected) {  in fuse_dev_queue_forget()
257 		fiq->forget_list_tail->next = forget;  in fuse_dev_queue_forget()
258 		fiq->forget_list_tail = forget;  in fuse_dev_queue_forget()
259 		fuse_dev_wake_and_unlock(fiq);  in fuse_dev_queue_forget()
262 		spin_unlock(&fiq->lock);  in fuse_dev_queue_forget()
266 static void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)  in fuse_dev_queue_interrupt()  argument
268 	spin_lock(&fiq->lock);  in fuse_dev_queue_interrupt()
270 		list_add_tail(&req->intr_entry, &fiq->interrupts);  in fuse_dev_queue_interrupt()
278 			spin_unlock(&fiq->lock);  in fuse_dev_queue_interrupt()
280 			fuse_dev_wake_and_unlock(fiq);  in fuse_dev_queue_interrupt()
283 		spin_unlock(&fiq->lock);  in fuse_dev_queue_interrupt()
287 static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req)  in fuse_dev_queue_req()  argument
289 	spin_lock(&fiq->lock);  in fuse_dev_queue_req()
290 	if (fiq->connected) {  in fuse_dev_queue_req()
292 			req->in.h.unique = fuse_get_unique_locked(fiq);  in fuse_dev_queue_req()
293 		list_add_tail(&req->list, &fiq->pending);  in fuse_dev_queue_req()
294 		fuse_dev_wake_and_unlock(fiq);  in fuse_dev_queue_req()
296 		spin_unlock(&fiq->lock);  in fuse_dev_queue_req()
310 static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req)  in fuse_send_one()  argument
316 	fiq->ops->send_req(fiq, req);  in fuse_send_one()
322 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_queue_forget()  local
327 	fiq->ops->send_forget(fiq, forget);  in fuse_queue_forget()
332 	struct fuse_iqueue *fiq = &fc->iq;  in flush_bg_queue()  local
341 		fuse_send_one(fiq, req);  in flush_bg_queue()
357 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_request_end()  local
369 		spin_lock(&fiq->lock);  in fuse_request_end()
371 		spin_unlock(&fiq->lock);  in fuse_request_end()
410 	struct fuse_iqueue *fiq = &req->fm->fc->iq;  in queue_interrupt()  local
416 	fiq->ops->send_interrupt(fiq, req);  in queue_interrupt()
424 	struct fuse_iqueue *fiq = &fc->iq;  in request_wait_answer()  local
448 		spin_lock(&fiq->lock);  in request_wait_answer()
452 			spin_unlock(&fiq->lock);  in request_wait_answer()
457 		spin_unlock(&fiq->lock);  in request_wait_answer()
469 	struct fuse_iqueue *fiq = &req->fm->fc->iq;  in __fuse_request_send()  local
476 	fuse_send_one(fiq, req);  in __fuse_request_send()
642 	struct fuse_iqueue *fiq = &fm->fc->iq;  in fuse_simple_notify_reply()  local
653 	fuse_send_one(fiq, req);  in fuse_simple_notify_reply()
1078 static int forget_pending(struct fuse_iqueue *fiq)  in forget_pending()  argument
1080 	return fiq->forget_list_head.next != NULL;  in forget_pending()
1083 static int request_pending(struct fuse_iqueue *fiq)  in request_pending()  argument
1085 	return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||  in request_pending()
1086 		forget_pending(fiq);  in request_pending()
1095  * Called with fiq->lock held, releases it
1097 static int fuse_read_interrupt(struct fuse_iqueue *fiq,  in fuse_read_interrupt()  argument
1100 __releases(fiq->lock)  in fuse_read_interrupt()
1115 	spin_unlock(&fiq->lock);  in fuse_read_interrupt()
1127 static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,  in fuse_dequeue_forget()  argument
1131 	struct fuse_forget_link *head = fiq->forget_list_head.next;  in fuse_dequeue_forget()
1138 	fiq->forget_list_head.next = *newhead;  in fuse_dequeue_forget()
1140 	if (fiq->forget_list_head.next == NULL)  in fuse_dequeue_forget()
1141 		fiq->forget_list_tail = &fiq->forget_list_head;  in fuse_dequeue_forget()
1149 static int fuse_read_single_forget(struct fuse_iqueue *fiq,  in fuse_read_single_forget()  argument
1152 __releases(fiq->lock)  in fuse_read_single_forget()
1155 	struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);  in fuse_read_single_forget()
1162 		.unique = fuse_get_unique_locked(fiq),  in fuse_read_single_forget()
1166 	spin_unlock(&fiq->lock);  in fuse_read_single_forget()
1182 static int fuse_read_batch_forget(struct fuse_iqueue *fiq,  in fuse_read_batch_forget()  argument
1184 __releases(fiq->lock)  in fuse_read_batch_forget()
1193 		.unique = fuse_get_unique_locked(fiq),  in fuse_read_batch_forget()
1198 		spin_unlock(&fiq->lock);  in fuse_read_batch_forget()
1203 	head = fuse_dequeue_forget(fiq, max_forgets, &count);  in fuse_read_batch_forget()
1204 	spin_unlock(&fiq->lock);  in fuse_read_batch_forget()
1231 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,  in fuse_read_forget()  argument
1234 __releases(fiq->lock)  in fuse_read_forget()
1236 	if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)  in fuse_read_forget()
1237 		return fuse_read_single_forget(fiq, cs, nbytes);  in fuse_read_forget()
1239 		return fuse_read_batch_forget(fiq, cs, nbytes);  in fuse_read_forget()
1256 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_dev_do_read()  local
1283 		spin_lock(&fiq->lock);  in fuse_dev_do_read()
1284 		if (!fiq->connected || request_pending(fiq))  in fuse_dev_do_read()
1286 		spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1290 		err = wait_event_interruptible_exclusive(fiq->waitq,  in fuse_dev_do_read()
1291 				!fiq->connected || request_pending(fiq));  in fuse_dev_do_read()
1296 	if (!fiq->connected) {  in fuse_dev_do_read()
1301 	if (!list_empty(&fiq->interrupts)) {  in fuse_dev_do_read()
1302 		req = list_entry(fiq->interrupts.next, struct fuse_req,  in fuse_dev_do_read()
1304 		return fuse_read_interrupt(fiq, cs, nbytes, req);  in fuse_dev_do_read()
1307 	if (forget_pending(fiq)) {  in fuse_dev_do_read()
1308 		if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)  in fuse_dev_do_read()
1309 			return fuse_read_forget(fc, fiq, cs, nbytes);  in fuse_dev_do_read()
1311 		if (fiq->forget_batch <= -8)  in fuse_dev_do_read()
1312 			fiq->forget_batch = 16;  in fuse_dev_do_read()
1315 	req = list_entry(fiq->pending.next, struct fuse_req, list);  in fuse_dev_do_read()
1318 	spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1385 	spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1844 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_resend()  local
1871 	spin_lock(&fiq->lock);  in fuse_resend()
1872 	if (!fiq->connected) {  in fuse_resend()
1873 		spin_unlock(&fiq->lock);  in fuse_resend()
1880 	list_splice(&to_queue, &fiq->pending);  in fuse_resend()
1881 	fuse_dev_wake_and_unlock(fiq);  in fuse_resend()
2175 	struct fuse_iqueue *fiq;  in fuse_dev_poll()  local
2181 	fiq = &fud->fc->iq;  in fuse_dev_poll()
2182 	poll_wait(file, &fiq->waitq, wait);  in fuse_dev_poll()
2184 	spin_lock(&fiq->lock);  in fuse_dev_poll()
2185 	if (!fiq->connected)  in fuse_dev_poll()
2187 	else if (request_pending(fiq))  in fuse_dev_poll()
2189 	spin_unlock(&fiq->lock);  in fuse_dev_poll()
2242 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_abort_conn()  local
2284 		spin_lock(&fiq->lock);  in fuse_abort_conn()
2285 		fiq->connected = 0;  in fuse_abort_conn()
2286 		list_for_each_entry(req, &fiq->pending, list)  in fuse_abort_conn()
2288 		list_splice_tail_init(&fiq->pending, &to_end);  in fuse_abort_conn()
2289 		while (forget_pending(fiq))  in fuse_abort_conn()
2290 			kfree(fuse_dequeue_forget(fiq, 1, NULL));  in fuse_abort_conn()
2291 		wake_up_all(&fiq->waitq);  in fuse_abort_conn()
2292 		spin_unlock(&fiq->lock);  in fuse_abort_conn()
2293 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_abort_conn()