194 static void fanotify_unhash_event(struct fsnotify_group *group,  in fanotify_unhash_event()  argument
197 	assert_spin_locked(&group->notification_lock);  in fanotify_unhash_event()
200 		 group, event, fanotify_event_hash_bucket(group, event));  in fanotify_unhash_event()
214 static struct fanotify_event *get_one_event(struct fsnotify_group *group,  in get_one_event()  argument
220 	unsigned int info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES);  in get_one_event()
222 	pr_debug("%s: group=%p count=%zd\n", __func__, group, count);  in get_one_event()
224 	spin_lock(&group->notification_lock);  in get_one_event()
225 	fsn_event = fsnotify_peek_first_event(group);  in get_one_event()
241 	fsnotify_remove_first_event(group);  in get_one_event()
245 		fanotify_unhash_event(group, event);  in get_one_event()
247 	spin_unlock(&group->notification_lock);  in get_one_event()
251 static int create_fd(struct fsnotify_group *group, const struct path *path,  in create_fd()  argument
257 	client_fd = get_unused_fd_flags(group->fanotify_data.f_flags);  in create_fd()
266 			       group->fanotify_data.f_flags | __FMODE_NONOTIFY,  in create_fd()
309 static void finish_permission_event(struct fsnotify_group *group,  in finish_permission_event()  argument
312 				    __releases(&group->notification_lock)  in finish_permission_event()
316 	assert_spin_locked(&group->notification_lock);  in finish_permission_event()
325 	spin_unlock(&group->notification_lock);  in finish_permission_event()
327 		fsnotify_destroy_event(group, &event->fae.fse);  in finish_permission_event()
330 static int process_access_response(struct fsnotify_group *group,  in process_access_response()  argument
342 		 group, fd, response, info, info_len);  in process_access_response()
359 	if ((response & FAN_AUDIT) && !FAN_GROUP_FLAG(group, FAN_ENABLE_AUDIT))  in process_access_response()
375 	spin_lock(&group->notification_lock);  in process_access_response()
376 	list_for_each_entry(event, &group->fanotify_data.access_list,  in process_access_response()
382 		finish_permission_event(group, event, response, &friar);  in process_access_response()
383 		wake_up(&group->fanotify_data.access_waitq);  in process_access_response()
386 	spin_unlock(&group->notification_lock);  in process_access_response()
656 static ssize_t copy_event_to_user(struct fsnotify_group *group,  in copy_event_to_user()  argument
663 	unsigned int info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES);  in copy_event_to_user()
668 	pr_debug("%s: group=%p event=%p\n", __func__, group, event);  in copy_event_to_user()
681 	if (FAN_GROUP_FLAG(group, FANOTIFY_UNPRIV) &&  in copy_event_to_user()
691 	if (!FAN_GROUP_FLAG(group, FANOTIFY_UNPRIV) &&  in copy_event_to_user()
693 		fd = create_fd(group, path, &f);  in copy_event_to_user()
706 		WARN_ON_ONCE(FAN_GROUP_FLAG(group, FAN_REPORT_TID));  in copy_event_to_user()
775 	struct fsnotify_group *group = file->private_data;  in fanotify_poll()  local
778 	poll_wait(file, &group->notification_waitq, wait);  in fanotify_poll()
779 	spin_lock(&group->notification_lock);  in fanotify_poll()
780 	if (!fsnotify_notify_queue_is_empty(group))  in fanotify_poll()
782 	spin_unlock(&group->notification_lock);  in fanotify_poll()
790 	struct fsnotify_group *group;  in fanotify_read()  local
797 	group = file->private_data;  in fanotify_read()
799 	pr_debug("%s: group=%p\n", __func__, group);  in fanotify_read()
801 	add_wait_queue(&group->notification_waitq, &wait);  in fanotify_read()
808 		event = get_one_event(group, count);  in fanotify_read()
830 		ret = copy_event_to_user(group, event, buf, count);  in fanotify_read()
846 			fsnotify_destroy_event(group, &event->fse);  in fanotify_read()
849 				spin_lock(&group->notification_lock);  in fanotify_read()
850 				finish_permission_event(group,  in fanotify_read()
852 				wake_up(&group->fanotify_data.access_waitq);  in fanotify_read()
854 				spin_lock(&group->notification_lock);  in fanotify_read()
856 					&group->fanotify_data.access_list);  in fanotify_read()
857 				spin_unlock(&group->notification_lock);  in fanotify_read()
865 	remove_wait_queue(&group->notification_waitq, &wait);  in fanotify_read()
875 	struct fsnotify_group *group;  in fanotify_write()  local
883 	group = file->private_data;  in fanotify_write()
885 	pr_debug("%s: group=%p count=%zu\n", __func__, group, count);  in fanotify_write()
895 	ret = process_access_response(group, &response, info_buf, info_len);  in fanotify_write()
906 	struct fsnotify_group *group = file->private_data;  in fanotify_release()  local
914 	fsnotify_group_stop_queueing(group);  in fanotify_release()
920 	spin_lock(&group->notification_lock);  in fanotify_release()
921 	while (!list_empty(&group->fanotify_data.access_list)) {  in fanotify_release()
924 		event = list_first_entry(&group->fanotify_data.access_list,  in fanotify_release()
927 		finish_permission_event(group, event, FAN_ALLOW, NULL);  in fanotify_release()
928 		spin_lock(&group->notification_lock);  in fanotify_release()
936 	while ((fsn_event = fsnotify_remove_first_event(group))) {  in fanotify_release()
940 			spin_unlock(&group->notification_lock);  in fanotify_release()
941 			fsnotify_destroy_event(group, fsn_event);  in fanotify_release()
943 			finish_permission_event(group, FANOTIFY_PERM(event),  in fanotify_release()
946 		spin_lock(&group->notification_lock);  in fanotify_release()
948 	spin_unlock(&group->notification_lock);  in fanotify_release()
951 	wake_up(&group->fanotify_data.access_waitq);  in fanotify_release()
954 	fsnotify_destroy_group(group);  in fanotify_release()
961 	struct fsnotify_group *group;  in fanotify_ioctl()  local
967 	group = file->private_data;  in fanotify_ioctl()
973 		spin_lock(&group->notification_lock);  in fanotify_ioctl()
974 		list_for_each_entry(fsn_event, &group->notification_list, list)  in fanotify_ioctl()
976 		spin_unlock(&group->notification_lock);  in fanotify_ioctl()
1078 static int fanotify_remove_mark(struct fsnotify_group *group,  in fanotify_remove_mark()  argument
1086 	fsnotify_group_lock(group);  in fanotify_remove_mark()
1087 	fsn_mark = fsnotify_find_mark(obj, obj_type, group);  in fanotify_remove_mark()
1089 		fsnotify_group_unlock(group);  in fanotify_remove_mark()
1099 	fsnotify_group_unlock(group);  in fanotify_remove_mark()
1176 static int fanotify_set_mark_fsid(struct fsnotify_group *group,  in fanotify_set_mark_fsid()  argument
1190 	if (list_empty(&group->marks_list))  in fanotify_set_mark_fsid()
1194 	list_for_each_entry(old, &group->marks_list, g_list) {  in fanotify_set_mark_fsid()
1227 static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group,  in fanotify_add_new_mark()  argument
1233 	struct ucounts *ucounts = group->fanotify_data.ucounts;  in fanotify_add_new_mark()
1243 	if (!FAN_GROUP_FLAG(group, FAN_UNLIMITED_MARKS) &&  in fanotify_add_new_mark()
1254 	fsnotify_init_mark(mark, group);  in fanotify_add_new_mark()
1260 		ret = fanotify_set_mark_fsid(group, mark, fsid);  in fanotify_add_new_mark()
1276 	if (!FAN_GROUP_FLAG(group, FAN_UNLIMITED_MARKS))  in fanotify_add_new_mark()
1281 static int fanotify_group_init_error_pool(struct fsnotify_group *group)  in fanotify_group_init_error_pool()  argument
1283 	if (mempool_initialized(&group->fanotify_data.error_events_pool))  in fanotify_group_init_error_pool()
1286 	return mempool_init_kmalloc_pool(&group->fanotify_data.error_events_pool,  in fanotify_group_init_error_pool()
1322 static int fanotify_add_mark(struct fsnotify_group *group,  in fanotify_add_mark()  argument
1331 	fsnotify_group_lock(group);  in fanotify_add_mark()
1332 	fsn_mark = fsnotify_find_mark(obj, obj_type, group);  in fanotify_add_mark()
1334 		fsn_mark = fanotify_add_new_mark(group, obj, obj_type,  in fanotify_add_mark()
1337 			fsnotify_group_unlock(group);  in fanotify_add_mark()
1355 		ret = fanotify_group_init_error_pool(group);  in fanotify_add_mark()
1365 	fsnotify_group_unlock(group);  in fanotify_add_mark()
1402 	struct fsnotify_group *group;  in SYSCALL_DEFINE2()  local
1482 	group = fsnotify_alloc_group(&fanotify_fsnotify_ops,  in SYSCALL_DEFINE2()
1484 	if (IS_ERR(group)) {  in SYSCALL_DEFINE2()
1485 		return PTR_ERR(group);  in SYSCALL_DEFINE2()
1489 	group->fanotify_data.ucounts = inc_ucount(current_user_ns(),  in SYSCALL_DEFINE2()
1492 	if (!group->fanotify_data.ucounts) {  in SYSCALL_DEFINE2()
1497 	group->fanotify_data.flags = flags | internal_flags;  in SYSCALL_DEFINE2()
1498 	group->memcg = get_mem_cgroup_from_mm(current->mm);  in SYSCALL_DEFINE2()
1500 	group->fanotify_data.merge_hash = fanotify_alloc_merge_hash();  in SYSCALL_DEFINE2()
1501 	if (!group->fanotify_data.merge_hash) {  in SYSCALL_DEFINE2()
1506 	group->overflow_event = fanotify_alloc_overflow_event();  in SYSCALL_DEFINE2()
1507 	if (unlikely(!group->overflow_event)) {  in SYSCALL_DEFINE2()
1514 	group->fanotify_data.f_flags = event_f_flags;  in SYSCALL_DEFINE2()
1515 	init_waitqueue_head(&group->fanotify_data.access_waitq);  in SYSCALL_DEFINE2()
1516 	INIT_LIST_HEAD(&group->fanotify_data.access_list);  in SYSCALL_DEFINE2()
1519 		group->priority = FSNOTIFY_PRIO_NORMAL;  in SYSCALL_DEFINE2()
1522 		group->priority = FSNOTIFY_PRIO_CONTENT;  in SYSCALL_DEFINE2()
1525 		group->priority = FSNOTIFY_PRIO_PRE_CONTENT;  in SYSCALL_DEFINE2()
1536 		group->max_events = UINT_MAX;  in SYSCALL_DEFINE2()
1538 		group->max_events = fanotify_max_queued_events;  in SYSCALL_DEFINE2()
1553 	fd = anon_inode_getfd("[fanotify]", &fanotify_fops, group, f_flags);  in SYSCALL_DEFINE2()
1560 	fsnotify_destroy_group(group);  in SYSCALL_DEFINE2()
1631 static int fanotify_events_supported(struct fsnotify_group *group,  in fanotify_events_supported()  argument
1637 	bool strict_dir_events = FAN_GROUP_FLAG(group, FAN_REPORT_TARGET_FID) ||  in fanotify_events_supported()
1684 	struct fsnotify_group *group;  in do_fanotify_mark()  local
1763 	group = fd_file(f)->private_data;  in do_fanotify_mark()
1772 	     FAN_GROUP_FLAG(group, FANOTIFY_UNPRIV)) &&  in do_fanotify_mark()
1781 	    group->priority < FSNOTIFY_PRIO_CONTENT)  in do_fanotify_mark()
1803 	fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS);  in do_fanotify_mark()
1819 			fsnotify_clear_vfsmount_marks_by_group(group);  in do_fanotify_mark()
1821 			fsnotify_clear_sb_marks_by_group(group);  in do_fanotify_mark()
1823 			fsnotify_clear_inode_marks_by_group(group);  in do_fanotify_mark()
1833 		ret = fanotify_events_supported(group, &path, mask, flags);  in do_fanotify_mark()
1896 		ret = fanotify_add_mark(group, obj, obj_type, mask, flags,  in do_fanotify_mark()
1900 		ret = fanotify_remove_mark(group, obj, obj_type, mask, flags,  in do_fanotify_mark()