34 static void release_registered_pages(struct tee_shm *shm)  in release_registered_pages()  argument
36 	if (shm->pages) {  in release_registered_pages()
37 		if (shm->flags & TEE_SHM_USER_MAPPED)  in release_registered_pages()
38 			unpin_user_pages(shm->pages, shm->num_pages);  in release_registered_pages()
40 			shm_put_kernel_pages(shm->pages, shm->num_pages);  in release_registered_pages()
42 		kfree(shm->pages);  in release_registered_pages()
46 static void tee_shm_release(struct tee_device *teedev, struct tee_shm *shm)  in tee_shm_release()  argument
48 	if (shm->flags & TEE_SHM_POOL) {  in tee_shm_release()
49 		teedev->pool->ops->free(teedev->pool, shm);  in tee_shm_release()
50 	} else if (shm->flags & TEE_SHM_DYNAMIC) {  in tee_shm_release()
51 		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);  in tee_shm_release()
55 				"unregister shm %p failed: %d", shm, rc);  in tee_shm_release()
57 		release_registered_pages(shm);  in tee_shm_release()
60 	teedev_ctx_put(shm->ctx);  in tee_shm_release()
62 	kfree(shm);  in tee_shm_release()
71 	struct tee_shm *shm;  in shm_alloc_helper()  local
84 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in shm_alloc_helper()
85 	if (!shm) {  in shm_alloc_helper()
90 	refcount_set(&shm->refcount, 1);  in shm_alloc_helper()
91 	shm->flags = flags;  in shm_alloc_helper()
92 	shm->id = id;  in shm_alloc_helper()
95 	 * We're assigning this as it is needed if the shm is to be  in shm_alloc_helper()
97 	 * to call teedev_ctx_get() or clear shm->ctx in case it's not  in shm_alloc_helper()
100 	shm->ctx = ctx;  in shm_alloc_helper()
102 	rc = teedev->pool->ops->alloc(teedev->pool, shm, size, align);  in shm_alloc_helper()
109 	return shm;  in shm_alloc_helper()
111 	kfree(shm);  in shm_alloc_helper()
133 	struct tee_shm *shm;  in tee_shm_alloc_user_buf()  local
143 	shm = shm_alloc_helper(ctx, size, PAGE_SIZE, flags, id);  in tee_shm_alloc_user_buf()
144 	if (IS_ERR(shm)) {  in tee_shm_alloc_user_buf()
148 		return shm;  in tee_shm_alloc_user_buf()
152 	ret = idr_replace(&teedev->idr, shm, id);  in tee_shm_alloc_user_buf()
155 		tee_shm_free(shm);  in tee_shm_alloc_user_buf()
159 	return shm;  in tee_shm_alloc_user_buf()
206 int tee_dyn_shm_alloc_helper(struct tee_shm *shm, size_t size, size_t align,  in tee_dyn_shm_alloc_helper()  argument
208 						 struct tee_shm *shm,  in tee_dyn_shm_alloc_helper()
222 	shm->kaddr = alloc_pages_exact(nr_pages * PAGE_SIZE,  in tee_dyn_shm_alloc_helper()
224 	if (!shm->kaddr)  in tee_dyn_shm_alloc_helper()
227 	shm->paddr = virt_to_phys(shm->kaddr);  in tee_dyn_shm_alloc_helper()
228 	shm->size = nr_pages * PAGE_SIZE;  in tee_dyn_shm_alloc_helper()
237 		pages[i] = virt_to_page((u8 *)shm->kaddr + i * PAGE_SIZE);  in tee_dyn_shm_alloc_helper()
239 	shm->pages = pages;  in tee_dyn_shm_alloc_helper()
240 	shm->num_pages = nr_pages;  in tee_dyn_shm_alloc_helper()
243 		rc = shm_register(shm->ctx, shm, pages, nr_pages,  in tee_dyn_shm_alloc_helper()
244 				  (unsigned long)shm->kaddr);  in tee_dyn_shm_alloc_helper()
251 	free_pages_exact(shm->kaddr, shm->size);  in tee_dyn_shm_alloc_helper()
252 	shm->kaddr = NULL;  in tee_dyn_shm_alloc_helper()
257 void tee_dyn_shm_free_helper(struct tee_shm *shm,  in tee_dyn_shm_free_helper()  argument
259 						   struct tee_shm *shm))  in tee_dyn_shm_free_helper()
262 		shm_unregister(shm->ctx, shm);  in tee_dyn_shm_free_helper()
263 	free_pages_exact(shm->kaddr, shm->size);  in tee_dyn_shm_free_helper()
264 	shm->kaddr = NULL;  in tee_dyn_shm_free_helper()
265 	kfree(shm->pages);  in tee_dyn_shm_free_helper()
266 	shm->pages = NULL;  in tee_dyn_shm_free_helper()
275 	struct tee_shm *shm;  in register_shm_helper()  local
293 	shm = kzalloc(sizeof(*shm), GFP_KERNEL);  in register_shm_helper()
294 	if (!shm) {  in register_shm_helper()
299 	refcount_set(&shm->refcount, 1);  in register_shm_helper()
300 	shm->flags = flags;  in register_shm_helper()
301 	shm->ctx = ctx;  in register_shm_helper()
302 	shm->id = id;  in register_shm_helper()
311 	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);  in register_shm_helper()
312 	if (!shm->pages) {  in register_shm_helper()
317 	len = iov_iter_extract_pages(iter, &shm->pages, LONG_MAX, num_pages, 0,  in register_shm_helper()
329 		shm_get_kernel_pages(shm->pages, num_pages);  in register_shm_helper()
331 	shm->offset = off;  in register_shm_helper()
332 	shm->size = len;  in register_shm_helper()
333 	shm->num_pages = num_pages;  in register_shm_helper()
335 	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,  in register_shm_helper()
336 					     shm->num_pages, start);  in register_shm_helper()
342 	return shm;  in register_shm_helper()
345 		unpin_user_pages(shm->pages, shm->num_pages);  in register_shm_helper()
347 		shm_put_kernel_pages(shm->pages, shm->num_pages);  in register_shm_helper()
349 	kfree(shm->pages);  in register_shm_helper()
351 	kfree(shm);  in register_shm_helper()
372 	struct tee_shm *shm;  in tee_shm_register_user_buf()  local
387 	shm = register_shm_helper(ctx, &iter, flags, id);  in tee_shm_register_user_buf()
388 	if (IS_ERR(shm)) {  in tee_shm_register_user_buf()
392 		return shm;  in tee_shm_register_user_buf()
396 	ret = idr_replace(&teedev->idr, shm, id);  in tee_shm_register_user_buf()
399 		tee_shm_free(shm);  in tee_shm_register_user_buf()
403 	return shm;  in tee_shm_register_user_buf()
439 	struct tee_shm *shm = filp->private_data;  in tee_shm_fop_mmap()  local
443 	if (shm->flags & TEE_SHM_USER_MAPPED)  in tee_shm_fop_mmap()
447 	if (vma->vm_pgoff + vma_pages(vma) > shm->size >> PAGE_SHIFT)  in tee_shm_fop_mmap()
450 	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,  in tee_shm_fop_mmap()
462  * @shm:	Shared memory handle
465 int tee_shm_get_fd(struct tee_shm *shm)  in tee_shm_get_fd()  argument
469 	if (shm->id < 0)  in tee_shm_get_fd()
473 	refcount_inc(&shm->refcount);  in tee_shm_get_fd()
474 	fd = anon_inode_getfd("tee_shm", &tee_shm_fops, shm, O_RDWR);  in tee_shm_get_fd()
476 		tee_shm_put(shm);  in tee_shm_get_fd()
482  * @shm:	Handle to shared memory to free
484 void tee_shm_free(struct tee_shm *shm)  in tee_shm_free()  argument
486 	tee_shm_put(shm);  in tee_shm_free()
492  * @shm:	Shared memory handle
497 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)  in tee_shm_get_va()  argument
499 	if (!shm->kaddr)  in tee_shm_get_va()
501 	if (offs >= shm->size)  in tee_shm_get_va()
503 	return (char *)shm->kaddr + offs;  in tee_shm_get_va()
509  * @shm:	Shared memory handle
515 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)  in tee_shm_get_pa()  argument
517 	if (offs >= shm->size)  in tee_shm_get_pa()
520 		*pa = shm->paddr + offs;  in tee_shm_get_pa()
535 	struct tee_shm *shm;  in tee_shm_get_from_id()  local
542 	shm = idr_find(&teedev->idr, id);  in tee_shm_get_from_id()
548 	if (!shm || shm->ctx != ctx)  in tee_shm_get_from_id()
549 		shm = ERR_PTR(-EINVAL);  in tee_shm_get_from_id()
551 		refcount_inc(&shm->refcount);  in tee_shm_get_from_id()
553 	return shm;  in tee_shm_get_from_id()
559  * @shm:	Shared memory handle
561 void tee_shm_put(struct tee_shm *shm)  in tee_shm_put()  argument
563 	struct tee_device *teedev = shm->ctx->teedev;  in tee_shm_put()
567 	if (refcount_dec_and_test(&shm->refcount)) {  in tee_shm_put()
574 		if (shm->id >= 0)  in tee_shm_put()
575 			idr_remove(&teedev->idr, shm->id);  in tee_shm_put()
581 		tee_shm_release(teedev, shm);  in tee_shm_put()