1 // SPDX-License-Identifier: GPL-2.0
5  * Pseudo-locking support built on top of Cache Allocation Technology (CAT)
43  * pseudo-locked regions.
55 	return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name);  in pseudo_lock_devnode()
64  * get_prefetch_disable_bits - prefetch disable bits of supported platforms
68  * pseudo-locking. This includes testing to ensure pseudo-locked regions
70  * as well as that these pseudo-locked regions can maintain their low cache
73  * After a platform has been validated to support pseudo-locking its
119  * pseudo_lock_minor_get - Obtain available minor number
134 		return -ENOSPC;  in pseudo_lock_minor_get()
143  * pseudo_lock_minor_release - Return minor number to available
152  * region_find_by_minor - Locate a pseudo-lock region by inode minor number
153  * @minor: The minor number of the device representing pseudo-locked region
156  * pseudo-locked region it belongs to. This is done by matching the minor
157  * number of the device to the pseudo-locked region it belongs.
159  * Minor numbers are assigned at the time a pseudo-locked region is associated
162  * Return: On success return pointer to resource group owning the pseudo-locked
170 		if (rdtgrp->plr && rdtgrp->plr->minor == minor) {  in region_find_by_minor()
179  * struct pseudo_lock_pm_req - A power management QoS request list entry
180  * @list:	Entry within the @pm_reqs list for a pseudo-locked region
192 	list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) {  in pseudo_lock_cstates_relax()
193 		dev_pm_qos_remove_request(&pm_req->req);  in pseudo_lock_cstates_relax()
194 		list_del(&pm_req->list);  in pseudo_lock_cstates_relax()
200  * pseudo_lock_cstates_constrain - Restrict cores from entering C6
201  * @plr: Pseudo-locked region
223 	for_each_cpu(cpu, &plr->d->hdr.cpu_mask) {  in pseudo_lock_cstates_constrain()
227 			ret = -ENOMEM;  in pseudo_lock_cstates_constrain()
231 					     &pm_req->req,  in pseudo_lock_cstates_constrain()
238 			ret = -1;  in pseudo_lock_cstates_constrain()
241 		list_add(&pm_req->list, &plr->pm_reqs);  in pseudo_lock_cstates_constrain()
252  * pseudo_lock_region_clear - Reset pseudo-lock region data
253  * @plr: pseudo-lock region
255  * All content of the pseudo-locked region is reset - any memory allocated
262 	plr->size = 0;  in pseudo_lock_region_clear()
263 	plr->line_size = 0;  in pseudo_lock_region_clear()
264 	kfree(plr->kmem);  in pseudo_lock_region_clear()
265 	plr->kmem = NULL;  in pseudo_lock_region_clear()
266 	plr->s = NULL;  in pseudo_lock_region_clear()
267 	if (plr->d)  in pseudo_lock_region_clear()
268 		plr->d->plr = NULL;  in pseudo_lock_region_clear()
269 	plr->d = NULL;  in pseudo_lock_region_clear()
270 	plr->cbm = 0;  in pseudo_lock_region_clear()
271 	plr->debugfs_dir = NULL;  in pseudo_lock_region_clear()
275  * pseudo_lock_region_init - Initialize pseudo-lock region information
276  * @plr: pseudo-lock region
278  * Called after user provided a schemata to be pseudo-locked. From the
281  * required for pseudo-locking is deduced from this data and &struct
283  * - size in bytes of the region to be pseudo-locked
284  * - cache line size to know the stride with which data needs to be accessed
285  *   to be pseudo-locked
286  * - a cpu associated with the cache instance on which the pseudo-locking
289  * Return: 0 on success, <0 on failure. Descriptive error will be written
294 	enum resctrl_scope scope = plr->s->res->ctrl_scope;  in pseudo_lock_region_init()
299 		return -ENODEV;  in pseudo_lock_region_init()
302 	plr->cpu = cpumask_first(&plr->d->hdr.cpu_mask);  in pseudo_lock_region_init()
304 	if (!cpu_online(plr->cpu)) {  in pseudo_lock_region_init()
306 				    plr->cpu);  in pseudo_lock_region_init()
307 		ret = -ENODEV;  in pseudo_lock_region_init()
311 	ci = get_cpu_cacheinfo_level(plr->cpu, scope);  in pseudo_lock_region_init()
313 		plr->line_size = ci->coherency_line_size;  in pseudo_lock_region_init()
314 		plr->size = rdtgroup_cbm_to_size(plr->s->res, plr->d, plr->cbm);  in pseudo_lock_region_init()
318 	ret = -1;  in pseudo_lock_region_init()
326  * pseudo_lock_init - Initialize a pseudo-lock region
327  * @rdtgrp: resource group to which new pseudo-locked region will belong
329  * A pseudo-locked region is associated with a resource group. When this
330  * association is created the pseudo-locked region is initialized. The
331  * details of the pseudo-locked region are not known at this time so only
342 		return -ENOMEM;  in pseudo_lock_init()
344 	init_waitqueue_head(&plr->lock_thread_wq);  in pseudo_lock_init()
345 	INIT_LIST_HEAD(&plr->pm_reqs);  in pseudo_lock_init()
346 	rdtgrp->plr = plr;  in pseudo_lock_init()
351  * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked
352  * @plr: pseudo-lock region
354  * Initialize the details required to set up the pseudo-locked region and
355  * allocate the contiguous memory that will be pseudo-locked to the cache.
357  * Return: 0 on success, <0 on failure.  Descriptive error will be written
372 	if (plr->size > KMALLOC_MAX_SIZE) {  in pseudo_lock_region_alloc()
374 		ret = -E2BIG;  in pseudo_lock_region_alloc()
378 	plr->kmem = kzalloc(plr->size, GFP_KERNEL);  in pseudo_lock_region_alloc()
379 	if (!plr->kmem) {  in pseudo_lock_region_alloc()
381 		ret = -ENOMEM;  in pseudo_lock_region_alloc()
394  * pseudo_lock_free - Free a pseudo-locked region
395  * @rdtgrp: resource group to which pseudo-locked region belonged
397  * The pseudo-locked region's resources have already been released, or not
405 	pseudo_lock_region_clear(rdtgrp->plr);  in pseudo_lock_free()
406 	kfree(rdtgrp->plr);  in pseudo_lock_free()
407 	rdtgrp->plr = NULL;  in pseudo_lock_free()
411  * pseudo_lock_fn - Load kernel memory into cache
412  * @_rdtgrp: resource group to which pseudo-lock region belongs
414  * This is the core pseudo-locking flow.
419  * with class of service set to the bitmask of the pseudo-locked region.
432 	struct pseudo_lock_region *plr = rdtgrp->plr;  in pseudo_lock_fn()
443 	 * pseudo-locking success rate when KASAN is active.  in pseudo_lock_fn()
456 	 * will get a cache hit in below loop from outside of pseudo-locked  in pseudo_lock_fn()
475 	 * being pseudo-locked is reached the hardware will not read beyond  in pseudo_lock_fn()
476 	 * the buffer and evict pseudo-locked memory read earlier from the  in pseudo_lock_fn()
483 	mem_r = plr->kmem;  in pseudo_lock_fn()
484 	size = plr->size;  in pseudo_lock_fn()
485 	line_size = plr->line_size;  in pseudo_lock_fn()
489 	 * pseudo-locked followed by reading of kernel memory to load it  in pseudo_lock_fn()
492 	__wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, rdtgrp->closid);  in pseudo_lock_fn()
495 	 * into cache region associated with just activated plr->closid.  in pseudo_lock_fn()
497 	 * - In first loop the cache region is shared with the page walker  in pseudo_lock_fn()
499 	 * - In the second loop the paging structure caches are used and  in pseudo_lock_fn()
526 	 * does not overlap with pseudo-locked region.  in pseudo_lock_fn()
530 	/* Re-enable the hardware prefetcher(s) */  in pseudo_lock_fn()
534 	plr->thread_done = 1;  in pseudo_lock_fn()
535 	wake_up_interruptible(&plr->lock_thread_wq);  in pseudo_lock_fn()
540  * rdtgroup_monitor_in_progress - Test if monitoring in progress
548 	return !list_empty(&rdtgrp->mon.crdtgrp_list);  in rdtgroup_monitor_in_progress()
552  * rdtgroup_locksetup_user_restrict - Restrict user access to group
555  * A resource group used for cache pseudo-locking cannot have cpus or tasks
602  * rdtgroup_locksetup_user_restore - Restore user access to group
649  * rdtgroup_locksetup_enter - Resource group enters locksetup mode
653  * to represent a pseudo-locked region and is in the process of being set
654  * up to do so. A resource group used for a pseudo-locked region would
657  * future. Monitoring of a pseudo-locked region is not allowed either.
659  * The above and more restrictions on a pseudo-locked region are checked
675 		rdt_last_cmd_puts("Cannot pseudo-lock default group\n");  in rdtgroup_locksetup_enter()
676 		return -EINVAL;  in rdtgroup_locksetup_enter()
680 	 * Cache Pseudo-locking not supported when CDP is enabled.  in rdtgroup_locksetup_enter()
684 	 * - When CDP is enabled two separate resources are exposed,  in rdtgroup_locksetup_enter()
686 	 *   The implication for pseudo-locking is that if a  in rdtgroup_locksetup_enter()
687 	 *   pseudo-locked region is created on a domain of one  in rdtgroup_locksetup_enter()
688 	 *   resource (eg. L3CODE), then a pseudo-locked region cannot  in rdtgroup_locksetup_enter()
691 	 *   pseudo-locked region involves a call to wbinvd that will  in rdtgroup_locksetup_enter()
693 	 * - Considering the previous, it may be possible to only  in rdtgroup_locksetup_enter()
694 	 *   expose one of the CDP resources to pseudo-locking and  in rdtgroup_locksetup_enter()
698 	 * - If only one region is exposed to pseudo-locking we should  in rdtgroup_locksetup_enter()
700 	 *   for pseudo-locking should take into account both resources.  in rdtgroup_locksetup_enter()
701 	 *   Similarly, if a pseudo-locked region is created in one  in rdtgroup_locksetup_enter()
708 		return -EINVAL;  in rdtgroup_locksetup_enter()
713 	 * platform does not support Cache Pseudo-Locking.  in rdtgroup_locksetup_enter()
717 		rdt_last_cmd_puts("Pseudo-locking not supported\n");  in rdtgroup_locksetup_enter()
718 		return -EINVAL;  in rdtgroup_locksetup_enter()
723 		return -EINVAL;  in rdtgroup_locksetup_enter()
728 		return -EINVAL;  in rdtgroup_locksetup_enter()
731 	if (!cpumask_empty(&rdtgrp->cpu_mask)) {  in rdtgroup_locksetup_enter()
733 		return -EINVAL;  in rdtgroup_locksetup_enter()
738 		return -EIO;  in rdtgroup_locksetup_enter()
743 		rdt_last_cmd_puts("Unable to init pseudo-lock region\n");  in rdtgroup_locksetup_enter()
750 	 * anymore when this group would be used for pseudo-locking. This  in rdtgroup_locksetup_enter()
753 	free_rmid(rdtgrp->closid, rdtgrp->mon.rmid);  in rdtgroup_locksetup_enter()
765  * rdtgroup_locksetup_exit - resource group exist locksetup mode
778 		ret = alloc_rmid(rdtgrp->closid);  in rdtgroup_locksetup_exit()
783 		rdtgrp->mon.rmid = ret;  in rdtgroup_locksetup_exit()
788 		free_rmid(rdtgrp->closid, rdtgrp->mon.rmid);  in rdtgroup_locksetup_exit()
797  * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked
803  * pseudo-locked region on @d.
808  * Return: true if @cbm overlaps with pseudo-locked region on @d, false
816 	if (d->plr) {  in rdtgroup_cbm_overlaps_pseudo_locked()
817 		cbm_len = d->plr->s->res->cache.cbm_len;  in rdtgroup_cbm_overlaps_pseudo_locked()
818 		cbm_b = d->plr->cbm;  in rdtgroup_cbm_overlaps_pseudo_locked()
826  * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy
829  * The setup of a pseudo-locked region affects all cache instances within
831  * pseudo-locked regions exist within a cache hierarchy to prevent any
832  * attempts to create new pseudo-locked regions in the same hierarchy.
834  * Return: true if a pseudo-locked region exists in the hierarchy of @d or
845 	/* Walking r->domains, ensure it can't race with cpuhp */  in rdtgroup_pseudo_locked_in_hierarchy()
852 	 * First determine which cpus have pseudo-locked regions  in rdtgroup_pseudo_locked_in_hierarchy()
856 		list_for_each_entry(d_i, &r->ctrl_domains, hdr.list) {  in rdtgroup_pseudo_locked_in_hierarchy()
857 			if (d_i->plr)  in rdtgroup_pseudo_locked_in_hierarchy()
859 					   &d_i->hdr.cpu_mask);  in rdtgroup_pseudo_locked_in_hierarchy()
864 	 * Next test if new pseudo-locked region would intersect with  in rdtgroup_pseudo_locked_in_hierarchy()
867 	if (cpumask_intersects(&d->hdr.cpu_mask, cpu_with_psl))  in rdtgroup_pseudo_locked_in_hierarchy()
875  * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory
876  * @_plr: pseudo-lock region to measure
902 	mem_r = READ_ONCE(plr->kmem);  in measure_cycles_lat_fn()
908 	for (i = 0; i < plr->size; i += 32) {  in measure_cycles_lat_fn()
915 		trace_pseudo_lock_mem_latency((u32)(end - start));  in measure_cycles_lat_fn()
919 	plr->thread_done = 1;  in measure_cycles_lat_fn()
920 	wake_up_interruptible(&plr->lock_thread_wq);  in measure_cycles_lat_fn()
969 	miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu,  in measure_residency_fn()
974 	hit_event = perf_event_create_kernel_counter(hit_attr, plr->cpu,  in measure_residency_fn()
1002 	 * interrupts disabled - it is thus safe to read the counter index.  in measure_residency_fn()
1006 	line_size = READ_ONCE(plr->line_size);  in measure_residency_fn()
1007 	mem_r = READ_ONCE(plr->kmem);  in measure_residency_fn()
1008 	size = READ_ONCE(plr->size);  in measure_residency_fn()
1011 	 * Read counter variables twice - first to load the instructions  in measure_residency_fn()
1018 	 * From SDM: Performing back-to-back fast reads are not guaranteed  in measure_residency_fn()
1054 	/* Re-enable hardware prefetchers */  in measure_residency_fn()
1065 	counts->miss_before = miss_before;  in measure_residency_fn()
1066 	counts->hits_before = hits_before;  in measure_residency_fn()
1067 	counts->miss_after  = miss_after;  in measure_residency_fn()
1068 	counts->hits_after  = hits_after;  in measure_residency_fn()
1078 	 * Non-architectural event for the Goldmont Microarchitecture  in measure_l2_residency()
1102 	trace_pseudo_lock_l2(counts.hits_after - counts.hits_before,  in measure_l2_residency()
1103 			     counts.miss_after - counts.miss_before);  in measure_l2_residency()
1105 	plr->thread_done = 1;  in measure_l2_residency()
1106 	wake_up_interruptible(&plr->lock_thread_wq);  in measure_l2_residency()
1142 	counts.miss_after -= counts.miss_before;  in measure_l3_residency()
1152 		counts.hits_after -= counts.hits_before;  in measure_l3_residency()
1154 		counts.hits_after -= min(counts.miss_after, counts.hits_after);  in measure_l3_residency()
1156 		counts.hits_after -= counts.hits_before;  in measure_l3_residency()
1161 	plr->thread_done = 1;  in measure_l3_residency()
1162 	wake_up_interruptible(&plr->lock_thread_wq);  in measure_l3_residency()
1167  * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region
1168  * @rdtgrp: Resource group to which the pseudo-locked region belongs.
1169  * @sel: Selector of which measurement to perform on a pseudo-locked region.
1171  * The measurement of latency to access a pseudo-locked region should be
1172  * done from a cpu that is associated with that pseudo-locked region.
1180 	struct pseudo_lock_region *plr = rdtgrp->plr;  in pseudo_lock_measure_cycles()
1183 	int ret = -1;  in pseudo_lock_measure_cycles()
1188 	if (rdtgrp->flags & RDT_DELETED) {  in pseudo_lock_measure_cycles()
1189 		ret = -ENODEV;  in pseudo_lock_measure_cycles()
1193 	if (!plr->d) {  in pseudo_lock_measure_cycles()
1194 		ret = -ENODEV;  in pseudo_lock_measure_cycles()
1198 	plr->thread_done = 0;  in pseudo_lock_measure_cycles()
1199 	cpu = cpumask_first(&plr->d->hdr.cpu_mask);  in pseudo_lock_measure_cycles()
1201 		ret = -ENODEV;  in pseudo_lock_measure_cycles()
1205 	plr->cpu = cpu;  in pseudo_lock_measure_cycles()
1232 	ret = wait_event_interruptible(plr->lock_thread_wq,  in pseudo_lock_measure_cycles()
1233 				       plr->thread_done == 1);  in pseudo_lock_measure_cycles()
1249 	struct rdtgroup *rdtgrp = file->private_data;  in pseudo_lock_measure_trigger()
1255 	buf_size = min(count, (sizeof(buf) - 1));  in pseudo_lock_measure_trigger()
1257 		return -EFAULT;  in pseudo_lock_measure_trigger()
1263 			return -EINVAL;  in pseudo_lock_measure_trigger()
1264 		ret = debugfs_file_get(file->f_path.dentry);  in pseudo_lock_measure_trigger()
1270 		debugfs_file_put(file->f_path.dentry);  in pseudo_lock_measure_trigger()
1283  * rdtgroup_pseudo_lock_create - Create a pseudo-locked region
1284  * @rdtgrp: resource group to which pseudo-lock region belongs
1286  * Called when a resource group in the pseudo-locksetup mode receives a
1287  * valid schemata that should be pseudo-locked. Since the resource group is
1288  * in pseudo-locksetup mode the &struct pseudo_lock_region has already been
1290  * occurs the resource group remains in the pseudo-locksetup mode with the
1292  * information and ready for the user to re-attempt pseudo-locking by
1295  * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0
1296  * on failure. Descriptive error will be written to last_cmd_status buffer.
1300 	struct pseudo_lock_region *plr = rdtgrp->plr;  in rdtgroup_pseudo_lock_create()
1312 		ret = -EINVAL;  in rdtgroup_pseudo_lock_create()
1316 	plr->thread_done = 0;  in rdtgroup_pseudo_lock_create()
1319 					cpu_to_node(plr->cpu),  in rdtgroup_pseudo_lock_create()
1320 					"pseudo_lock/%u", plr->cpu);  in rdtgroup_pseudo_lock_create()
1327 	kthread_bind(thread, plr->cpu);  in rdtgroup_pseudo_lock_create()
1330 	ret = wait_event_interruptible(plr->lock_thread_wq,  in rdtgroup_pseudo_lock_create()
1331 				       plr->thread_done == 1);  in rdtgroup_pseudo_lock_create()
1339 		 * empty pseudo-locking loop.  in rdtgroup_pseudo_lock_create()
1353 	 * pseudo-locked region will still be here on return.  in rdtgroup_pseudo_lock_create()
1356 	 * deadlock with the mm->mmap_lock which is obtained in the  in rdtgroup_pseudo_lock_create()
1363 		plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name,  in rdtgroup_pseudo_lock_create()
1365 		if (!IS_ERR_OR_NULL(plr->debugfs_dir))  in rdtgroup_pseudo_lock_create()
1367 					    plr->debugfs_dir, rdtgrp,  in rdtgroup_pseudo_lock_create()
1373 			    rdtgrp, "%s", rdtgrp->kn->name);  in rdtgroup_pseudo_lock_create()
1384 	/* We released the mutex - check if group was removed while we did so */  in rdtgroup_pseudo_lock_create()
1385 	if (rdtgrp->flags & RDT_DELETED) {  in rdtgroup_pseudo_lock_create()
1386 		ret = -ENODEV;  in rdtgroup_pseudo_lock_create()
1390 	plr->minor = new_minor;  in rdtgroup_pseudo_lock_create()
1392 	rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED;  in rdtgroup_pseudo_lock_create()
1393 	closid_free(rdtgrp->closid);  in rdtgroup_pseudo_lock_create()
1403 	debugfs_remove_recursive(plr->debugfs_dir);  in rdtgroup_pseudo_lock_create()
1414  * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region
1415  * @rdtgrp: resource group to which the pseudo-locked region belongs
1417  * The removal of a pseudo-locked region can be initiated when the resource
1420  * not go back to pseudo-locksetup mode before it is removed, instead it is
1429 	struct pseudo_lock_region *plr = rdtgrp->plr;  in rdtgroup_pseudo_lock_remove()
1431 	if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {  in rdtgroup_pseudo_lock_remove()
1433 		 * Default group cannot be a pseudo-locked region so we can  in rdtgroup_pseudo_lock_remove()
1436 		closid_free(rdtgrp->closid);  in rdtgroup_pseudo_lock_remove()
1441 	debugfs_remove_recursive(rdtgrp->plr->debugfs_dir);  in rdtgroup_pseudo_lock_remove()
1442 	device_destroy(&pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor));  in rdtgroup_pseudo_lock_remove()
1443 	pseudo_lock_minor_release(plr->minor);  in rdtgroup_pseudo_lock_remove()
1458 		return -ENODEV;  in pseudo_lock_dev_open()
1461 	filp->private_data = rdtgrp;  in pseudo_lock_dev_open()
1462 	atomic_inc(&rdtgrp->waitcount);  in pseudo_lock_dev_open()
1463 	/* Perform a non-seekable open - llseek is not supported */  in pseudo_lock_dev_open()
1464 	filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);  in pseudo_lock_dev_open()
1476 	rdtgrp = filp->private_data;  in pseudo_lock_dev_release()
1480 		return -ENODEV;  in pseudo_lock_dev_release()
1482 	filp->private_data = NULL;  in pseudo_lock_dev_release()
1483 	atomic_dec(&rdtgrp->waitcount);  in pseudo_lock_dev_release()
1491 	return -EINVAL;  in pseudo_lock_dev_mremap()
1500 	unsigned long vsize = vma->vm_end - vma->vm_start;  in pseudo_lock_dev_mmap()
1501 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;  in pseudo_lock_dev_mmap()
1509 	rdtgrp = filp->private_data;  in pseudo_lock_dev_mmap()
1513 		return -ENODEV;  in pseudo_lock_dev_mmap()
1516 	plr = rdtgrp->plr;  in pseudo_lock_dev_mmap()
1518 	if (!plr->d) {  in pseudo_lock_dev_mmap()
1520 		return -ENODEV;  in pseudo_lock_dev_mmap()
1525 	 * with the pseudo-locked region. If this is not the case the task  in pseudo_lock_dev_mmap()
1527 	 * pseudo-locked region.  in pseudo_lock_dev_mmap()
1529 	if (!cpumask_subset(current->cpus_ptr, &plr->d->hdr.cpu_mask)) {  in pseudo_lock_dev_mmap()
1531 		return -EINVAL;  in pseudo_lock_dev_mmap()
1534 	physical = __pa(plr->kmem) >> PAGE_SHIFT;  in pseudo_lock_dev_mmap()
1535 	psize = plr->size - off;  in pseudo_lock_dev_mmap()
1537 	if (off > plr->size) {  in pseudo_lock_dev_mmap()
1539 		return -ENOSPC;  in pseudo_lock_dev_mmap()
1544 	 * do not allow copy-on-write mapping.  in pseudo_lock_dev_mmap()
1546 	if (!(vma->vm_flags & VM_SHARED)) {  in pseudo_lock_dev_mmap()
1548 		return -EINVAL;  in pseudo_lock_dev_mmap()
1553 		return -ENOSPC;  in pseudo_lock_dev_mmap()
1556 	memset(plr->kmem + off, 0, vsize);  in pseudo_lock_dev_mmap()
1558 	if (remap_pfn_range(vma, vma->vm_start, physical + vma->vm_pgoff,  in pseudo_lock_dev_mmap()
1559 			    vsize, vma->vm_page_prot)) {  in pseudo_lock_dev_mmap()
1561 		return -EAGAIN;  in pseudo_lock_dev_mmap()
1563 	vma->vm_ops = &pseudo_mmap_ops;  in pseudo_lock_dev_mmap()