1  // SPDX-License-Identifier: GPL-2.0-only
2  /*
3   * Copyright 2023 Red Hat
4   */
5  
6  #include "completion.h"
7  
8  #include <linux/kernel.h>
9  
10  #include "logger.h"
11  #include "permassert.h"
12  
13  #include "status-codes.h"
14  #include "types.h"
15  #include "vio.h"
16  #include "vdo.h"
17  
18  /**
19   * DOC: vdo completions.
20   *
21   * Most of vdo's data structures are lock free, each either belonging to a single "zone," or
22   * divided into a number of zones whose accesses to the structure do not overlap. During normal
23   * operation, at most one thread will be operating in any given zone. Each zone has a
24   * vdo_work_queue which holds vdo_completions that are to be run in that zone. A completion may
25   * only be enqueued on one queue or operating in a single zone at a time.
26   *
27   * At each step of a multi-threaded operation, the completion performing the operation is given a
28   * callback, error handler, and thread id for the next step. A completion is "run" when it is
29   * operating on the correct thread (as specified by its callback_thread_id). If the value of its
30   * "result" field is an error (i.e. not VDO_SUCCESS), the function in its "error_handler" will be
31   * invoked. If the error_handler is NULL, or there is no error, the function set as its "callback"
32   * will be invoked. Generally, a completion will not be run directly, but rather will be
33   * "launched." In this case, it will check whether it is operating on the correct thread. If it is,
34   * it will run immediately. Otherwise, it will be enqueue on the vdo_work_queue associated with the
35   * completion's "callback_thread_id". When it is dequeued, it will be on the correct thread, and
36   * will get run. In some cases, the completion should get queued instead of running immediately,
37   * even if it is being launched from the correct thread. This is usually in cases where there is a
38   * long chain of callbacks, all on the same thread, which could overflow the stack. In such cases,
39   * the completion's "requeue" field should be set to true. Doing so will skip the current thread
40   * check and simply enqueue the completion.
41   *
42   * A completion may be "finished," in which case its "complete" field will be set to true before it
43   * is next run. It is a bug to attempt to set the result or re-finish a finished completion.
44   * Because a completion's fields are not safe to examine from any thread other than the one on
45   * which the completion is currently operating, this field is used only to aid in detecting
46   * programming errors. It can not be used for cross-thread checking on the status of an operation.
47   * A completion must be "reset" before it can be reused after it has been finished. Resetting will
48   * also clear any error from the result field.
49   **/
50  
vdo_initialize_completion(struct vdo_completion * completion,struct vdo * vdo,enum vdo_completion_type type)51  void vdo_initialize_completion(struct vdo_completion *completion,
52  			       struct vdo *vdo,
53  			       enum vdo_completion_type type)
54  {
55  	memset(completion, 0, sizeof(*completion));
56  	completion->vdo = vdo;
57  	completion->type = type;
58  	vdo_reset_completion(completion);
59  }
60  
assert_incomplete(struct vdo_completion * completion)61  static inline void assert_incomplete(struct vdo_completion *completion)
62  {
63  	VDO_ASSERT_LOG_ONLY(!completion->complete, "completion is not complete");
64  }
65  
66  /**
67   * vdo_set_completion_result() - Set the result of a completion.
68   *
69   * Older errors will not be masked.
70   */
vdo_set_completion_result(struct vdo_completion * completion,int result)71  void vdo_set_completion_result(struct vdo_completion *completion, int result)
72  {
73  	assert_incomplete(completion);
74  	if (completion->result == VDO_SUCCESS)
75  		completion->result = result;
76  }
77  
78  /**
79   * vdo_launch_completion_with_priority() - Run or enqueue a completion.
80   * @priority: The priority at which to enqueue the completion.
81   *
82   * If called on the correct thread (i.e. the one specified in the completion's callback_thread_id
83   * field) and not marked for requeue, the completion will be run immediately. Otherwise, the
84   * completion will be enqueued on the specified thread.
85   */
vdo_launch_completion_with_priority(struct vdo_completion * completion,enum vdo_completion_priority priority)86  void vdo_launch_completion_with_priority(struct vdo_completion *completion,
87  					 enum vdo_completion_priority priority)
88  {
89  	thread_id_t callback_thread = completion->callback_thread_id;
90  
91  	if (completion->requeue || (callback_thread != vdo_get_callback_thread_id())) {
92  		vdo_enqueue_completion(completion, priority);
93  		return;
94  	}
95  
96  	vdo_run_completion(completion);
97  }
98  
99  /** vdo_finish_completion() - Mark a completion as complete and then launch it. */
vdo_finish_completion(struct vdo_completion * completion)100  void vdo_finish_completion(struct vdo_completion *completion)
101  {
102  	assert_incomplete(completion);
103  	completion->complete = true;
104  	if (completion->callback != NULL)
105  		vdo_launch_completion(completion);
106  }
107  
vdo_enqueue_completion(struct vdo_completion * completion,enum vdo_completion_priority priority)108  void vdo_enqueue_completion(struct vdo_completion *completion,
109  			    enum vdo_completion_priority priority)
110  {
111  	struct vdo *vdo = completion->vdo;
112  	thread_id_t thread_id = completion->callback_thread_id;
113  
114  	if (VDO_ASSERT(thread_id < vdo->thread_config.thread_count,
115  		       "thread_id %u (completion type %d) is less than thread count %u",
116  		       thread_id, completion->type,
117  		       vdo->thread_config.thread_count) != VDO_SUCCESS)
118  		BUG();
119  
120  	completion->requeue = false;
121  	completion->priority = priority;
122  	completion->my_queue = NULL;
123  	vdo_enqueue_work_queue(vdo->threads[thread_id].queue, completion);
124  }
125  
126  /**
127   * vdo_requeue_completion_if_needed() - Requeue a completion if not called on the specified thread.
128   *
129   * Return: True if the completion was requeued; callers may not access the completion in this case.
130   */
vdo_requeue_completion_if_needed(struct vdo_completion * completion,thread_id_t callback_thread_id)131  bool vdo_requeue_completion_if_needed(struct vdo_completion *completion,
132  				      thread_id_t callback_thread_id)
133  {
134  	if (vdo_get_callback_thread_id() == callback_thread_id)
135  		return false;
136  
137  	completion->callback_thread_id = callback_thread_id;
138  	vdo_enqueue_completion(completion, VDO_WORK_Q_DEFAULT_PRIORITY);
139  	return true;
140  }
141