1 // SPDX-License-Identifier: GPL-2.0
4  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
8 #include <linux/blk-mq.h>
10 #include <linux/module.h>
37 	struct nvme_ctrl	ctrl;  member
42 static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)  in to_loop_ctrl()  argument
44 	return container_of(ctrl, struct nvme_loop_ctrl, ctrl);  in to_loop_ctrl()
54 	struct nvme_loop_ctrl	*ctrl;  member
65 static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
71 	return queue - queue->ctrl->queues;  in nvme_loop_queue_idx()
78 	sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT);  in nvme_loop_complete_rq()
87 		return queue->ctrl->admin_tag_set.tags[queue_idx];  in nvme_loop_tagset()
88 	return queue->ctrl->tag_set.tags[queue_idx - 1];  in nvme_loop_tagset()
94 		container_of(req->sq, struct nvme_loop_queue, nvme_sq);  in nvme_loop_queue_response()
95 	struct nvme_completion *cqe = req->cqe;  in nvme_loop_queue_response()
104 				     cqe->command_id))) {  in nvme_loop_queue_response()
105 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,  in nvme_loop_queue_response()
106 				&cqe->result);  in nvme_loop_queue_response()
110 		rq = nvme_find_rq(nvme_loop_tagset(queue), cqe->command_id);  in nvme_loop_queue_response()
112 			dev_err(queue->ctrl->ctrl.device,  in nvme_loop_queue_response()
114 				cqe->command_id, nvme_loop_queue_idx(queue));  in nvme_loop_queue_response()
118 		if (!nvme_try_complete_req(rq, cqe->status, cqe->result))  in nvme_loop_queue_response()
128 	iod->req.execute(&iod->req);  in nvme_loop_execute_work()
134 	struct nvme_ns *ns = hctx->queue->queuedata;  in nvme_loop_queue_rq()
135 	struct nvme_loop_queue *queue = hctx->driver_data;  in nvme_loop_queue_rq()
136 	struct request *req = bd->rq;  in nvme_loop_queue_rq()
138 	bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags);  in nvme_loop_queue_rq()
141 	if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready))  in nvme_loop_queue_rq()
142 		return nvme_fail_nonready_command(&queue->ctrl->ctrl, req);  in nvme_loop_queue_rq()
149 	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;  in nvme_loop_queue_rq()
150 	iod->req.port = queue->ctrl->port;  in nvme_loop_queue_rq()
151 	if (!nvmet_req_init(&iod->req, &queue->nvme_cq,  in nvme_loop_queue_rq()
152 			&queue->nvme_sq, &nvme_loop_ops))  in nvme_loop_queue_rq()
156 		iod->sg_table.sgl = iod->first_sgl;  in nvme_loop_queue_rq()
157 		if (sg_alloc_table_chained(&iod->sg_table,  in nvme_loop_queue_rq()
159 				iod->sg_table.sgl, NVME_INLINE_SG_CNT)) {  in nvme_loop_queue_rq()
164 		iod->req.sg = iod->sg_table.sgl;  in nvme_loop_queue_rq()
165 		iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl);  in nvme_loop_queue_rq()
166 		iod->req.transfer_len = blk_rq_payload_bytes(req);  in nvme_loop_queue_rq()
169 	queue_work(nvmet_wq, &iod->work);  in nvme_loop_queue_rq()
175 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);  in nvme_loop_submit_async_event()  local
176 	struct nvme_loop_queue *queue = &ctrl->queues[0];  in nvme_loop_submit_async_event()
177 	struct nvme_loop_iod *iod = &ctrl->async_event_iod;  in nvme_loop_submit_async_event()
179 	memset(&iod->cmd, 0, sizeof(iod->cmd));  in nvme_loop_submit_async_event()
180 	iod->cmd.common.opcode = nvme_admin_async_event;  in nvme_loop_submit_async_event()
181 	iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;  in nvme_loop_submit_async_event()
182 	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;  in nvme_loop_submit_async_event()
184 	if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq,  in nvme_loop_submit_async_event()
186 		dev_err(ctrl->ctrl.device, "failed async event work\n");  in nvme_loop_submit_async_event()
190 	queue_work(nvmet_wq, &iod->work);  in nvme_loop_submit_async_event()
193 static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,  in nvme_loop_init_iod()  argument
196 	iod->req.cmd = &iod->cmd;  in nvme_loop_init_iod()
197 	iod->req.cqe = &iod->cqe;  in nvme_loop_init_iod()
198 	iod->queue = &ctrl->queues[queue_idx];  in nvme_loop_init_iod()
199 	INIT_WORK(&iod->work, nvme_loop_execute_work);  in nvme_loop_init_iod()
207 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(set->driver_data);  in nvme_loop_init_request()  local
210 	nvme_req(req)->ctrl = &ctrl->ctrl;  in nvme_loop_init_request()
211 	nvme_req(req)->cmd = &iod->cmd;  in nvme_loop_init_request()
212 	return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),  in nvme_loop_init_request()
213 			(set == &ctrl->tag_set) ? hctx_idx + 1 : 0);  in nvme_loop_init_request()
221 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);  in nvme_loop_init_hctx()  local
222 	struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];  in nvme_loop_init_hctx()
224 	BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);  in nvme_loop_init_hctx()
234 	hctx->driver_data = queue;  in nvme_loop_init_hctx()
241 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);  in nvme_loop_init_admin_hctx()  local
242 	struct nvme_loop_queue *queue = &ctrl->queues[0];  in nvme_loop_init_admin_hctx()
246 	hctx->driver_data = queue;  in nvme_loop_init_admin_hctx()
264 static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)  in nvme_loop_destroy_admin_queue()  argument
266 	if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))  in nvme_loop_destroy_admin_queue()
273 	nvme_unquiesce_admin_queue(&ctrl->ctrl);  in nvme_loop_destroy_admin_queue()
275 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);  in nvme_loop_destroy_admin_queue()
276 	nvme_remove_admin_tag_set(&ctrl->ctrl);  in nvme_loop_destroy_admin_queue()
281 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);  in nvme_loop_free_ctrl()  local
283 	if (list_empty(&ctrl->list))  in nvme_loop_free_ctrl()
287 	list_del(&ctrl->list);  in nvme_loop_free_ctrl()
290 	if (nctrl->tagset)  in nvme_loop_free_ctrl()
292 	kfree(ctrl->queues);  in nvme_loop_free_ctrl()
293 	nvmf_free_options(nctrl->opts);  in nvme_loop_free_ctrl()
295 	kfree(ctrl);  in nvme_loop_free_ctrl()
298 static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_destroy_io_queues()  argument
302 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {  in nvme_loop_destroy_io_queues()
303 		clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);  in nvme_loop_destroy_io_queues()
304 		nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);  in nvme_loop_destroy_io_queues()
306 	ctrl->ctrl.queue_count = 1;  in nvme_loop_destroy_io_queues()
312 	nvme_unquiesce_io_queues(&ctrl->ctrl);  in nvme_loop_destroy_io_queues()
315 static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_init_io_queues()  argument
317 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;  in nvme_loop_init_io_queues()
321 	nr_io_queues = min(opts->nr_io_queues, num_online_cpus());  in nvme_loop_init_io_queues()
322 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);  in nvme_loop_init_io_queues()
326 	dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);  in nvme_loop_init_io_queues()
329 		ctrl->queues[i].ctrl = ctrl;  in nvme_loop_init_io_queues()
330 		ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);  in nvme_loop_init_io_queues()
334 		ctrl->ctrl.queue_count++;  in nvme_loop_init_io_queues()
340 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_init_io_queues()
344 static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_connect_io_queues()  argument
348 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {  in nvme_loop_connect_io_queues()
349 		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);  in nvme_loop_connect_io_queues()
352 		set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);  in nvme_loop_connect_io_queues()
358 static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)  in nvme_loop_configure_admin_queue()  argument
362 	ctrl->queues[0].ctrl = ctrl;  in nvme_loop_configure_admin_queue()
363 	error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);  in nvme_loop_configure_admin_queue()
366 	ctrl->ctrl.queue_count = 1;  in nvme_loop_configure_admin_queue()
368 	error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,  in nvme_loop_configure_admin_queue()
376 	clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags);  in nvme_loop_configure_admin_queue()
378 	error = nvmf_connect_admin_queue(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
382 	set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);  in nvme_loop_configure_admin_queue()
384 	error = nvme_enable_ctrl(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
388 	ctrl->ctrl.max_hw_sectors =  in nvme_loop_configure_admin_queue()
389 		(NVME_LOOP_MAX_SEGMENTS - 1) << PAGE_SECTORS_SHIFT;  in nvme_loop_configure_admin_queue()
391 	nvme_unquiesce_admin_queue(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
393 	error = nvme_init_ctrl_finish(&ctrl->ctrl, false);  in nvme_loop_configure_admin_queue()
400 	clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);  in nvme_loop_configure_admin_queue()
401 	nvme_remove_admin_tag_set(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
403 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);  in nvme_loop_configure_admin_queue()
407 static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)  in nvme_loop_shutdown_ctrl()  argument
409 	if (ctrl->ctrl.queue_count > 1) {  in nvme_loop_shutdown_ctrl()
410 		nvme_quiesce_io_queues(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
411 		nvme_cancel_tagset(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
412 		nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_shutdown_ctrl()
415 	nvme_quiesce_admin_queue(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
416 	if (nvme_ctrl_state(&ctrl->ctrl) == NVME_CTRL_LIVE)  in nvme_loop_shutdown_ctrl()
417 		nvme_disable_ctrl(&ctrl->ctrl, true);  in nvme_loop_shutdown_ctrl()
419 	nvme_cancel_admin_tagset(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
420 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_shutdown_ctrl()
423 static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)  in nvme_loop_delete_ctrl_host()  argument
425 	nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));  in nvme_loop_delete_ctrl_host()
430 	struct nvme_loop_ctrl *ctrl;  in nvme_loop_delete_ctrl()  local
433 	list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {  in nvme_loop_delete_ctrl()
434 		if (ctrl->ctrl.cntlid == nctrl->cntlid)  in nvme_loop_delete_ctrl()
435 			nvme_delete_ctrl(&ctrl->ctrl);  in nvme_loop_delete_ctrl()
442 	struct nvme_loop_ctrl *ctrl =  in nvme_loop_reset_ctrl_work()  local
443 		container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);  in nvme_loop_reset_ctrl_work()
446 	nvme_stop_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
447 	nvme_loop_shutdown_ctrl(ctrl);  in nvme_loop_reset_ctrl_work()
449 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {  in nvme_loop_reset_ctrl_work()
450 		enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
454 			/* state change failure for non-deleted ctrl? */  in nvme_loop_reset_ctrl_work()
459 	ret = nvme_loop_configure_admin_queue(ctrl);  in nvme_loop_reset_ctrl_work()
463 	ret = nvme_loop_init_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
467 	ret = nvme_loop_connect_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
471 	blk_mq_update_nr_hw_queues(&ctrl->tag_set,  in nvme_loop_reset_ctrl_work()
472 			ctrl->ctrl.queue_count - 1);  in nvme_loop_reset_ctrl_work()
474 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))  in nvme_loop_reset_ctrl_work()
477 	nvme_start_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
482 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
484 	nvme_quiesce_admin_queue(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
485 	nvme_cancel_admin_tagset(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
486 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_reset_ctrl_work()
488 	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");  in nvme_loop_reset_ctrl_work()
489 	nvme_uninit_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
494 	.module			= THIS_MODULE,
505 static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_create_io_queues()  argument
509 	ret = nvme_loop_init_io_queues(ctrl);  in nvme_loop_create_io_queues()
513 	ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,  in nvme_loop_create_io_queues()
520 	ret = nvme_loop_connect_io_queues(ctrl);  in nvme_loop_create_io_queues()
527 	nvme_remove_io_tag_set(&ctrl->ctrl);  in nvme_loop_create_io_queues()
529 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_create_io_queues()
533 static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)  in nvme_loop_find_port()  argument
540 		if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&  in nvme_loop_find_port()
541 		    strcmp(ctrl->opts->traddr, p->disc_addr.traddr))  in nvme_loop_find_port()
553 	struct nvme_loop_ctrl *ctrl;  in nvme_loop_create_ctrl()  local
556 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);  in nvme_loop_create_ctrl()
557 	if (!ctrl)  in nvme_loop_create_ctrl()
558 		return ERR_PTR(-ENOMEM);  in nvme_loop_create_ctrl()
559 	ctrl->ctrl.opts = opts;  in nvme_loop_create_ctrl()
560 	INIT_LIST_HEAD(&ctrl->list);  in nvme_loop_create_ctrl()
562 	INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);  in nvme_loop_create_ctrl()
564 	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,  in nvme_loop_create_ctrl()
567 		kfree(ctrl);  in nvme_loop_create_ctrl()
571 	ret = nvme_add_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
575 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))  in nvme_loop_create_ctrl()
578 	ret = -ENOMEM;  in nvme_loop_create_ctrl()
580 	ctrl->ctrl.kato = opts->kato;  in nvme_loop_create_ctrl()
581 	ctrl->port = nvme_loop_find_port(&ctrl->ctrl);  in nvme_loop_create_ctrl()
583 	ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),  in nvme_loop_create_ctrl()
585 	if (!ctrl->queues)  in nvme_loop_create_ctrl()
588 	ret = nvme_loop_configure_admin_queue(ctrl);  in nvme_loop_create_ctrl()
592 	if (opts->queue_size > ctrl->ctrl.maxcmd) {  in nvme_loop_create_ctrl()
594 		dev_warn(ctrl->ctrl.device,  in nvme_loop_create_ctrl()
595 			"queue_size %zu > ctrl maxcmd %u, clamping down\n",  in nvme_loop_create_ctrl()
596 			opts->queue_size, ctrl->ctrl.maxcmd);  in nvme_loop_create_ctrl()
597 		opts->queue_size = ctrl->ctrl.maxcmd;  in nvme_loop_create_ctrl()
599 	ctrl->ctrl.sqsize = opts->queue_size - 1;  in nvme_loop_create_ctrl()
601 	if (opts->nr_io_queues) {  in nvme_loop_create_ctrl()
602 		ret = nvme_loop_create_io_queues(ctrl);  in nvme_loop_create_ctrl()
607 	nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);  in nvme_loop_create_ctrl()
609 	dev_info(ctrl->ctrl.device,  in nvme_loop_create_ctrl()
610 		 "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);  in nvme_loop_create_ctrl()
612 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))  in nvme_loop_create_ctrl()
616 	list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);  in nvme_loop_create_ctrl()
619 	nvme_start_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
621 	return &ctrl->ctrl;  in nvme_loop_create_ctrl()
624 	nvme_quiesce_admin_queue(&ctrl->ctrl);  in nvme_loop_create_ctrl()
625 	nvme_cancel_admin_tagset(&ctrl->ctrl);  in nvme_loop_create_ctrl()
626 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_create_ctrl()
628 	kfree(ctrl->queues);  in nvme_loop_create_ctrl()
630 	nvme_uninit_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
632 	nvme_put_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
635 		ret = -EIO;  in nvme_loop_create_ctrl()
642 	list_add_tail(&port->entry, &nvme_loop_ports);  in nvme_loop_add_port()
650 	list_del_init(&port->entry);  in nvme_loop_remove_port()
673 	.module		= THIS_MODULE,
695 	struct nvme_loop_ctrl *ctrl, *next;  in nvme_loop_cleanup_module()  local
701 	list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)  in nvme_loop_cleanup_module()
702 		nvme_delete_ctrl(&ctrl->ctrl);  in nvme_loop_cleanup_module()
713 MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */