47 	nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0x00000003);  in ga100_chan_stop()
55 	const int gfid = 0;  in ga100_chan_start()
57 	nvkm_wr32(device, runl->chan + (chan->id * 4), 0x00000002);  in ga100_chan_start()
58 	nvkm_wr32(device, runl->addr + 0x0090, (gfid << 16) | chan->id); /* INTERNAL_DOORBELL. */  in ga100_chan_start()
66 	nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0xffffffff);  in ga100_chan_unbind()
75 	nvkm_wo32(chan->inst, 0x010, 0x0000face);  in ga100_chan_ramfc_write()
76 	nvkm_wo32(chan->inst, 0x030, 0x7ffff902);  in ga100_chan_ramfc_write()
77 	nvkm_wo32(chan->inst, 0x048, lower_32_bits(offset));  in ga100_chan_ramfc_write()
78 	nvkm_wo32(chan->inst, 0x04c, upper_32_bits(offset) | (limit2 << 16));  in ga100_chan_ramfc_write()
79 	nvkm_wo32(chan->inst, 0x084, 0x20400000);  in ga100_chan_ramfc_write()
80 	nvkm_wo32(chan->inst, 0x094, 0x30000000 | devm);  in ga100_chan_ramfc_write()
81 	nvkm_wo32(chan->inst, 0x0e4, priv ? 0x00000020 : 0x00000000);  in ga100_chan_ramfc_write()
82 	nvkm_wo32(chan->inst, 0x0e8, chan->id);  in ga100_chan_ramfc_write()
83 	nvkm_wo32(chan->inst, 0x0f4, 0x00001000 | (priv ? 0x00000100 : 0x00000000));  in ga100_chan_ramfc_write()
84 	nvkm_wo32(chan->inst, 0x0f8, 0x80000000 | chan->cgrp->runl->nonstall.vector);  in ga100_chan_ramfc_write()
85 	nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000);  in ga100_chan_ramfc_write()
87 	return 0;  in ga100_chan_ramfc_write()
93 	.devm = 0xfff,
114 	nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x01000000 | cgrp->id);  in ga100_cgrp_preempt()
127 	u32 stat = nvkm_rd32(device, runl->addr + 0x200 + engn->id * 0x40);  in ga100_engn_cxid()
132 	switch ((stat & 0x0000e000) >> 13) {  in ga100_engn_cxid()
133 	case 0 /* INVALID */: return -ENODEV;  in ga100_engn_cxid()
135 	case 5 /*    SAVE */: return (stat & 0x00000fff);  in ga100_engn_cxid()
136 	case 6 /*    LOAD */: return (stat & 0x0fff0000) >> 16;  in ga100_engn_cxid()
139 			return (stat & 0x0fff0000) >> 16;  in ga100_engn_cxid()
140 		return (stat & 0x00000fff);  in ga100_engn_cxid()
181 	return !(nvkm_rd32(device, 0x04015c + (runq->id * 0x800)) & 0x0000e000);  in ga100_runq_idle()
188 	u32 inte = nvkm_rd32(device, 0x040180 + (runq->id * 0x800));  in ga100_runq_intr_1()
189 	u32 intr = nvkm_rd32(device, 0x040148 + (runq->id * 0x800));  in ga100_runq_intr_1()
197 	if (stat & 0x80000000) {  in ga100_runq_intr_1()
198 		u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask;  in ga100_runq_intr_1()
209 		nvkm_mask(device, 0x0400ac + (runq->id * 0x800), 0x00030000, 0x00030000);  in ga100_runq_intr_1()
210 		stat &= ~0x80000000;  in ga100_runq_intr_1()
215 		nvkm_wr32(device, 0x0401a0 + (runq->id * 0x800), stat);  in ga100_runq_intr_1()
218 	nvkm_wr32(device, 0x040148 + (runq->id * 0x800), intr);  in ga100_runq_intr_1()
226 	u32 inte = nvkm_rd32(device, 0x040170 + (runq->id * 0x800));  in ga100_runq_intr_0()
227 	u32 intr = nvkm_rd32(device, 0x040108 + (runq->id * 0x800));  in ga100_runq_intr_0()
236 	if (stat & 0xc6afe000) {  in ga100_runq_intr_0()
237 		u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask;  in ga100_runq_intr_0()
248 		stat &= ~0xc6afe000;  in ga100_runq_intr_0()
253 		nvkm_wr32(device, 0x040190 + (runq->id * 0x800), stat);  in ga100_runq_intr_0()
256 	nvkm_wr32(device, 0x040108 + (runq->id * 0x800), intr);  in ga100_runq_intr_0()
274 	nvkm_wr32(device, 0x040108 + (runq->id * 0x800), 0xffffffff); /* INTR_0 */  in ga100_runq_init()
275 	nvkm_wr32(device, 0x040148 + (runq->id * 0x800), 0xffffffff); /* INTR_1 */  in ga100_runq_init()
276 	nvkm_wr32(device, 0x040170 + (runq->id * 0x800), 0xffffffff); /* INTR_0_EN_SET_TREE */  in ga100_runq_init()
277 	nvkm_wr32(device, 0x040180 + (runq->id * 0x800), 0xffffffff); /* INTR_1_EN_SET_TREE */  in ga100_runq_init()
290 	return nvkm_rd32(runl->fifo->engine.subdev.device, runl->addr + 0x098) & 0x00100000;  in ga100_runl_preempt_pending()
296 	nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x00000000);  in ga100_runl_preempt()
302 	nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000000);  in ga100_runl_allow()
308 	nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000001);  in ga100_runl_block()
316 	return nvkm_rd32(device, runl->addr + 0x08c) & 0x00008000;  in ga100_runl_pending()
325 	nvkm_wr32(device, runl->addr + 0x080, lower_32_bits(addr));  in ga100_runl_commit()
326 	nvkm_wr32(device, runl->addr + 0x084, upper_32_bits(addr));  in ga100_runl_commit()
327 	nvkm_wr32(device, runl->addr + 0x088, count);  in ga100_runl_commit()
336 	u32 inte = nvkm_rd32(device, runl->addr + 0x120);  in ga100_runl_intr()
337 	u32 intr = nvkm_rd32(device, runl->addr + 0x100);  in ga100_runl_intr()
346 	if (stat & 0x00000007) {  in ga100_runl_intr()
348 			info = nvkm_rd32(device, runl->addr + 0x224 + (engn->id * 0x40));  in ga100_runl_intr()
352 			nvkm_wr32(device, runl->addr + 0x100, BIT(engn->id));  in ga100_runl_intr()
357 	if (stat & 0x00000300) {  in ga100_runl_intr()
358 		nvkm_wr32(device, runl->addr + 0x100, stat & 0x00000300);  in ga100_runl_intr()
359 		stat &= ~0x00000300;  in ga100_runl_intr()
362 	if (stat & 0x00010000) {  in ga100_runl_intr()
363 		if (runl->runq[0]) {  in ga100_runl_intr()
364 			if (runl->runq[0]->func->intr(runl->runq[0], runl))  in ga100_runl_intr()
365 				stat &= ~0x00010000;  in ga100_runl_intr()
369 	if (stat & 0x00020000) {  in ga100_runl_intr()
372 				stat &= ~0x00020000;  in ga100_runl_intr()
378 		nvkm_wr32(device, runl->addr + 0x140, stat);  in ga100_runl_intr()
381 	nvkm_wr32(device, runl->addr + 0x180, 0x00000001);  in ga100_runl_intr()
388 	nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x300, 0x80000000, 0x00000000);  in ga100_runl_fini()
401 	nvkm_wr32(device, runl->addr + 0x088, 0x00000000);  in ga100_runl_init()
405 	nvkm_mask(device, runl->addr + 0x300, 0x80000000, 0x80000000);  in ga100_runl_init()
407 	nvkm_wr32(device, runl->addr + 0x100, 0xffffffff); /* INTR_0 */  in ga100_runl_init()
408 	nvkm_wr32(device, runl->addr + 0x140, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(0) */  in ga100_runl_init()
409 	nvkm_wr32(device, runl->addr + 0x120, 0x000f1307); /* INTR_0_EN_SET_TREE(0) */  in ga100_runl_init()
410 	nvkm_wr32(device, runl->addr + 0x148, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(1) */  in ga100_runl_init()
411 	nvkm_wr32(device, runl->addr + 0x128, 0x00000000); /* INTR_0_EN_SET_TREE(1) */  in ga100_runl_init()
414 	for (i = 0; i < runl->runq_nr; i++) {  in ga100_runl_init()
446 	u32 chcfg  = nvkm_rd32(device, addr + 0x004);  in ga100_runl_new()
447 	u32 chnum  = 1 << (chcfg & 0x0000000f);  in ga100_runl_new()
448 	u32 chaddr = (chcfg & 0xfffffff0);  in ga100_runl_new()
449 	u32 dbcfg  = nvkm_rd32(device, addr + 0x008);  in ga100_runl_new()
450 	u32 vector = nvkm_rd32(device, addr + 0x160);  in ga100_runl_new()
459 	for (i = 0; i < 2; i++) {  in ga100_runl_new()
460 		u32 pbcfg = nvkm_rd32(device, addr + 0x010 + (i * 0x04));  in ga100_runl_new()
461 		if (pbcfg & 0x80000000) {  in ga100_runl_new()
463 				nvkm_runq_new(fifo, ((pbcfg & 0x03fffc00) - 0x040000) / 0x800);  in ga100_runl_new()
474 		if (tdev->engine < 0) {  in ga100_runl_new()
496 	ret = nvkm_inth_add(&device->vfn->intr, vector & 0x00000fff, NVKM_INTR_PRIO_NORMAL,  in ga100_runl_new()
505 	return 0;  in ga100_runl_new()
521 	struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);  in ga100_fifo_nonstall_block()
530 	struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);  in ga100_fifo_nonstall_allow()
547 	int ret, nr = 0;  in ga100_fifo_nonstall_ctor()
557 		if (runl->nonstall.vector < 0) {  in ga100_fifo_nonstall_ctor()
579 	int id = 0, ret;  in ga100_fifo_runl_ctor()
581 	nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) {  in ga100_fifo_runl_ctor()
594 	return 0;  in ga100_fifo_runl_ctor()
607 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &ga100_cgrp, .force = true },
608 	.chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_A }, &ga100_chan },