1  // SPDX-License-Identifier: GPL-2.0
2  #include "bcachefs.h"
3  #include "btree_iter.h"
4  #include "error.h"
5  #include "journal.h"
6  #include "recovery_passes.h"
7  #include "super.h"
8  #include "thread_with_file.h"
9  
10  #define FSCK_ERR_RATELIMIT_NR	10
11  
bch2_inconsistent_error(struct bch_fs * c)12  bool bch2_inconsistent_error(struct bch_fs *c)
13  {
14  	set_bit(BCH_FS_error, &c->flags);
15  
16  	switch (c->opts.errors) {
17  	case BCH_ON_ERROR_continue:
18  		return false;
19  	case BCH_ON_ERROR_fix_safe:
20  	case BCH_ON_ERROR_ro:
21  		if (bch2_fs_emergency_read_only(c))
22  			bch_err(c, "inconsistency detected - emergency read only at journal seq %llu",
23  				journal_cur_seq(&c->journal));
24  		return true;
25  	case BCH_ON_ERROR_panic:
26  		panic(bch2_fmt(c, "panic after error"));
27  		return true;
28  	default:
29  		BUG();
30  	}
31  }
32  
bch2_topology_error(struct bch_fs * c)33  int bch2_topology_error(struct bch_fs *c)
34  {
35  	set_bit(BCH_FS_topology_error, &c->flags);
36  	if (!test_bit(BCH_FS_fsck_running, &c->flags)) {
37  		bch2_inconsistent_error(c);
38  		return -BCH_ERR_btree_need_topology_repair;
39  	} else {
40  		return bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology) ?:
41  			-BCH_ERR_btree_node_read_validate_error;
42  	}
43  }
44  
bch2_fatal_error(struct bch_fs * c)45  void bch2_fatal_error(struct bch_fs *c)
46  {
47  	if (bch2_fs_emergency_read_only(c))
48  		bch_err(c, "fatal error - emergency read only");
49  }
50  
bch2_io_error_work(struct work_struct * work)51  void bch2_io_error_work(struct work_struct *work)
52  {
53  	struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work);
54  	struct bch_fs *c = ca->fs;
55  	bool dev;
56  
57  	down_write(&c->state_lock);
58  	dev = bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_ro,
59  				    BCH_FORCE_IF_DEGRADED);
60  	if (dev
61  	    ? __bch2_dev_set_state(c, ca, BCH_MEMBER_STATE_ro,
62  				  BCH_FORCE_IF_DEGRADED)
63  	    : bch2_fs_emergency_read_only(c))
64  		bch_err(ca,
65  			"too many IO errors, setting %s RO",
66  			dev ? "device" : "filesystem");
67  	up_write(&c->state_lock);
68  }
69  
bch2_io_error(struct bch_dev * ca,enum bch_member_error_type type)70  void bch2_io_error(struct bch_dev *ca, enum bch_member_error_type type)
71  {
72  	atomic64_inc(&ca->errors[type]);
73  	//queue_work(system_long_wq, &ca->io_error_work);
74  }
75  
76  enum ask_yn {
77  	YN_NO,
78  	YN_YES,
79  	YN_ALLNO,
80  	YN_ALLYES,
81  };
82  
parse_yn_response(char * buf)83  static enum ask_yn parse_yn_response(char *buf)
84  {
85  	buf = strim(buf);
86  
87  	if (strlen(buf) == 1)
88  		switch (buf[0]) {
89  		case 'n':
90  			return YN_NO;
91  		case 'y':
92  			return YN_YES;
93  		case 'N':
94  			return YN_ALLNO;
95  		case 'Y':
96  			return YN_ALLYES;
97  		}
98  	return -1;
99  }
100  
101  #ifdef __KERNEL__
bch2_fsck_ask_yn(struct bch_fs * c,struct btree_trans * trans)102  static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)
103  {
104  	struct stdio_redirect *stdio = c->stdio;
105  
106  	if (c->stdio_filter && c->stdio_filter != current)
107  		stdio = NULL;
108  
109  	if (!stdio)
110  		return YN_NO;
111  
112  	if (trans)
113  		bch2_trans_unlock(trans);
114  
115  	unsigned long unlock_long_at = trans ? jiffies + HZ * 2 : 0;
116  	darray_char line = {};
117  	int ret;
118  
119  	do {
120  		unsigned long t;
121  		bch2_print(c, " (y,n, or Y,N for all errors of this type) ");
122  rewait:
123  		t = unlock_long_at
124  			? max_t(long, unlock_long_at - jiffies, 0)
125  			: MAX_SCHEDULE_TIMEOUT;
126  
127  		int r = bch2_stdio_redirect_readline_timeout(stdio, &line, t);
128  		if (r == -ETIME) {
129  			bch2_trans_unlock_long(trans);
130  			unlock_long_at = 0;
131  			goto rewait;
132  		}
133  
134  		if (r < 0) {
135  			ret = YN_NO;
136  			break;
137  		}
138  
139  		darray_last(line) = '\0';
140  	} while ((ret = parse_yn_response(line.data)) < 0);
141  
142  	darray_exit(&line);
143  	return ret;
144  }
145  #else
146  
147  #include "tools-util.h"
148  
bch2_fsck_ask_yn(struct bch_fs * c,struct btree_trans * trans)149  static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)
150  {
151  	char *buf = NULL;
152  	size_t buflen = 0;
153  	int ret;
154  
155  	do {
156  		fputs(" (y,n, or Y,N for all errors of this type) ", stdout);
157  		fflush(stdout);
158  
159  		if (getline(&buf, &buflen, stdin) < 0)
160  			die("error reading from standard input");
161  	} while ((ret = parse_yn_response(buf)) < 0);
162  
163  	free(buf);
164  	return ret;
165  }
166  
167  #endif
168  
fsck_err_get(struct bch_fs * c,const char * fmt)169  static struct fsck_err_state *fsck_err_get(struct bch_fs *c, const char *fmt)
170  {
171  	struct fsck_err_state *s;
172  
173  	if (!test_bit(BCH_FS_fsck_running, &c->flags))
174  		return NULL;
175  
176  	list_for_each_entry(s, &c->fsck_error_msgs, list)
177  		if (s->fmt == fmt) {
178  			/*
179  			 * move it to the head of the list: repeated fsck errors
180  			 * are common
181  			 */
182  			list_move(&s->list, &c->fsck_error_msgs);
183  			return s;
184  		}
185  
186  	s = kzalloc(sizeof(*s), GFP_NOFS);
187  	if (!s) {
188  		if (!c->fsck_alloc_msgs_err)
189  			bch_err(c, "kmalloc err, cannot ratelimit fsck errs");
190  		c->fsck_alloc_msgs_err = true;
191  		return NULL;
192  	}
193  
194  	INIT_LIST_HEAD(&s->list);
195  	s->fmt = fmt;
196  	list_add(&s->list, &c->fsck_error_msgs);
197  	return s;
198  }
199  
200  /* s/fix?/fixing/ s/recreate?/recreating/ */
prt_actioning(struct printbuf * out,const char * action)201  static void prt_actioning(struct printbuf *out, const char *action)
202  {
203  	unsigned len = strlen(action);
204  
205  	BUG_ON(action[len - 1] != '?');
206  	--len;
207  
208  	if (action[len - 1] == 'e')
209  		--len;
210  
211  	prt_bytes(out, action, len);
212  	prt_str(out, "ing");
213  }
214  
215  static const u8 fsck_flags_extra[] = {
216  #define x(t, n, flags)		[BCH_FSCK_ERR_##t] = flags,
217  	BCH_SB_ERRS()
218  #undef x
219  };
220  
__bch2_fsck_err(struct bch_fs * c,struct btree_trans * trans,enum bch_fsck_flags flags,enum bch_sb_error_id err,const char * fmt,...)221  int __bch2_fsck_err(struct bch_fs *c,
222  		  struct btree_trans *trans,
223  		  enum bch_fsck_flags flags,
224  		  enum bch_sb_error_id err,
225  		  const char *fmt, ...)
226  {
227  	struct fsck_err_state *s = NULL;
228  	va_list args;
229  	bool print = true, suppressing = false, inconsistent = false;
230  	struct printbuf buf = PRINTBUF, *out = &buf;
231  	int ret = -BCH_ERR_fsck_ignore;
232  	const char *action_orig = "fix?", *action = action_orig;
233  
234  	might_sleep();
235  
236  	if (!WARN_ON(err >= ARRAY_SIZE(fsck_flags_extra)))
237  		flags |= fsck_flags_extra[err];
238  
239  	if (!c)
240  		c = trans->c;
241  
242  	/*
243  	 * Ugly: if there's a transaction in the current task it has to be
244  	 * passed in to unlock if we prompt for user input.
245  	 *
246  	 * But, plumbing a transaction and transaction restarts into
247  	 * bkey_validate() is problematic.
248  	 *
249  	 * So:
250  	 * - make all bkey errors AUTOFIX, they're simple anyways (we just
251  	 *   delete the key)
252  	 * - and we don't need to warn if we're not prompting
253  	 */
254  	WARN_ON((flags & FSCK_CAN_FIX) &&
255  		!(flags & FSCK_AUTOFIX) &&
256  		!trans &&
257  		bch2_current_has_btree_trans(c));
258  
259  	if ((flags & FSCK_CAN_FIX) &&
260  	    test_bit(err, c->sb.errors_silent))
261  		return -BCH_ERR_fsck_fix;
262  
263  	bch2_sb_error_count(c, err);
264  
265  	va_start(args, fmt);
266  	prt_vprintf(out, fmt, args);
267  	va_end(args);
268  
269  	/* Custom fix/continue/recreate/etc.? */
270  	if (out->buf[out->pos - 1] == '?') {
271  		const char *p = strrchr(out->buf, ',');
272  		if (p) {
273  			out->pos = p - out->buf;
274  			action = kstrdup(p + 2, GFP_KERNEL);
275  			if (!action) {
276  				ret = -ENOMEM;
277  				goto err;
278  			}
279  		}
280  	}
281  
282  	mutex_lock(&c->fsck_error_msgs_lock);
283  	s = fsck_err_get(c, fmt);
284  	if (s) {
285  		/*
286  		 * We may be called multiple times for the same error on
287  		 * transaction restart - this memoizes instead of asking the user
288  		 * multiple times for the same error:
289  		 */
290  		if (s->last_msg && !strcmp(buf.buf, s->last_msg)) {
291  			ret = s->ret;
292  			mutex_unlock(&c->fsck_error_msgs_lock);
293  			goto err;
294  		}
295  
296  		kfree(s->last_msg);
297  		s->last_msg = kstrdup(buf.buf, GFP_KERNEL);
298  		if (!s->last_msg) {
299  			mutex_unlock(&c->fsck_error_msgs_lock);
300  			ret = -ENOMEM;
301  			goto err;
302  		}
303  
304  		if (c->opts.ratelimit_errors &&
305  		    !(flags & FSCK_NO_RATELIMIT) &&
306  		    s->nr >= FSCK_ERR_RATELIMIT_NR) {
307  			if (s->nr == FSCK_ERR_RATELIMIT_NR)
308  				suppressing = true;
309  			else
310  				print = false;
311  		}
312  
313  		s->nr++;
314  	}
315  
316  #ifdef BCACHEFS_LOG_PREFIX
317  	if (!strncmp(fmt, "bcachefs:", 9))
318  		prt_printf(out, bch2_log_msg(c, ""));
319  #endif
320  
321  	if ((flags & FSCK_CAN_FIX) &&
322  	    (flags & FSCK_AUTOFIX) &&
323  	    (c->opts.errors == BCH_ON_ERROR_continue ||
324  	     c->opts.errors == BCH_ON_ERROR_fix_safe)) {
325  		prt_str(out, ", ");
326  		prt_actioning(out, action);
327  		ret = -BCH_ERR_fsck_fix;
328  	} else if (!test_bit(BCH_FS_fsck_running, &c->flags)) {
329  		if (c->opts.errors != BCH_ON_ERROR_continue ||
330  		    !(flags & (FSCK_CAN_FIX|FSCK_CAN_IGNORE))) {
331  			prt_str(out, ", shutting down");
332  			inconsistent = true;
333  			ret = -BCH_ERR_fsck_errors_not_fixed;
334  		} else if (flags & FSCK_CAN_FIX) {
335  			prt_str(out, ", ");
336  			prt_actioning(out, action);
337  			ret = -BCH_ERR_fsck_fix;
338  		} else {
339  			prt_str(out, ", continuing");
340  			ret = -BCH_ERR_fsck_ignore;
341  		}
342  	} else if (c->opts.fix_errors == FSCK_FIX_exit) {
343  		prt_str(out, ", exiting");
344  		ret = -BCH_ERR_fsck_errors_not_fixed;
345  	} else if (flags & FSCK_CAN_FIX) {
346  		int fix = s && s->fix
347  			? s->fix
348  			: c->opts.fix_errors;
349  
350  		if (fix == FSCK_FIX_ask) {
351  			prt_str(out, ", ");
352  			prt_str(out, action);
353  
354  			if (bch2_fs_stdio_redirect(c))
355  				bch2_print(c, "%s", out->buf);
356  			else
357  				bch2_print_string_as_lines(KERN_ERR, out->buf);
358  			print = false;
359  
360  			int ask = bch2_fsck_ask_yn(c, trans);
361  
362  			if (trans) {
363  				ret = bch2_trans_relock(trans);
364  				if (ret) {
365  					mutex_unlock(&c->fsck_error_msgs_lock);
366  					goto err;
367  				}
368  			}
369  
370  			if (ask >= YN_ALLNO && s)
371  				s->fix = ask == YN_ALLNO
372  					? FSCK_FIX_no
373  					: FSCK_FIX_yes;
374  
375  			ret = ask & 1
376  				? -BCH_ERR_fsck_fix
377  				: -BCH_ERR_fsck_ignore;
378  		} else if (fix == FSCK_FIX_yes ||
379  			   (c->opts.nochanges &&
380  			    !(flags & FSCK_CAN_IGNORE))) {
381  			prt_str(out, ", ");
382  			prt_actioning(out, action);
383  			ret = -BCH_ERR_fsck_fix;
384  		} else {
385  			prt_str(out, ", not ");
386  			prt_actioning(out, action);
387  		}
388  	} else if (flags & FSCK_NEED_FSCK) {
389  		prt_str(out, " (run fsck to correct)");
390  	} else {
391  		prt_str(out, " (repair unimplemented)");
392  	}
393  
394  	if (ret == -BCH_ERR_fsck_ignore &&
395  	    (c->opts.fix_errors == FSCK_FIX_exit ||
396  	     !(flags & FSCK_CAN_IGNORE)))
397  		ret = -BCH_ERR_fsck_errors_not_fixed;
398  
399  	bool exiting =
400  		test_bit(BCH_FS_fsck_running, &c->flags) &&
401  		(ret != -BCH_ERR_fsck_fix &&
402  		 ret != -BCH_ERR_fsck_ignore);
403  
404  	if (exiting)
405  		print = true;
406  
407  	if (print) {
408  		if (bch2_fs_stdio_redirect(c))
409  			bch2_print(c, "%s\n", out->buf);
410  		else
411  			bch2_print_string_as_lines(KERN_ERR, out->buf);
412  	}
413  
414  	if (exiting)
415  		bch_err(c, "Unable to continue, halting");
416  	else if (suppressing)
417  		bch_err(c, "Ratelimiting new instances of previous error");
418  
419  	if (s)
420  		s->ret = ret;
421  
422  	mutex_unlock(&c->fsck_error_msgs_lock);
423  
424  	if (inconsistent)
425  		bch2_inconsistent_error(c);
426  
427  	if (ret == -BCH_ERR_fsck_fix) {
428  		set_bit(BCH_FS_errors_fixed, &c->flags);
429  	} else {
430  		set_bit(BCH_FS_errors_not_fixed, &c->flags);
431  		set_bit(BCH_FS_error, &c->flags);
432  	}
433  err:
434  	if (action != action_orig)
435  		kfree(action);
436  	printbuf_exit(&buf);
437  	return ret;
438  }
439  
__bch2_bkey_fsck_err(struct bch_fs * c,struct bkey_s_c k,enum bch_validate_flags validate_flags,enum bch_sb_error_id err,const char * fmt,...)440  int __bch2_bkey_fsck_err(struct bch_fs *c,
441  			 struct bkey_s_c k,
442  			 enum bch_validate_flags validate_flags,
443  			 enum bch_sb_error_id err,
444  			 const char *fmt, ...)
445  {
446  	if (validate_flags & BCH_VALIDATE_silent)
447  		return -BCH_ERR_fsck_delete_bkey;
448  
449  	unsigned fsck_flags = 0;
450  	if (!(validate_flags & (BCH_VALIDATE_write|BCH_VALIDATE_commit)))
451  		fsck_flags |= FSCK_AUTOFIX|FSCK_CAN_FIX;
452  
453  	struct printbuf buf = PRINTBUF;
454  	va_list args;
455  
456  	prt_str(&buf, "invalid bkey ");
457  	bch2_bkey_val_to_text(&buf, c, k);
458  	prt_str(&buf, "\n  ");
459  	va_start(args, fmt);
460  	prt_vprintf(&buf, fmt, args);
461  	va_end(args);
462  	prt_str(&buf, ": delete?");
463  
464  	int ret = __bch2_fsck_err(c, NULL, fsck_flags, err, "%s", buf.buf);
465  	printbuf_exit(&buf);
466  	return ret;
467  }
468  
bch2_flush_fsck_errs(struct bch_fs * c)469  void bch2_flush_fsck_errs(struct bch_fs *c)
470  {
471  	struct fsck_err_state *s, *n;
472  
473  	mutex_lock(&c->fsck_error_msgs_lock);
474  
475  	list_for_each_entry_safe(s, n, &c->fsck_error_msgs, list) {
476  		if (s->ratelimited && s->last_msg)
477  			bch_err(c, "Saw %llu errors like:\n    %s", s->nr, s->last_msg);
478  
479  		list_del(&s->list);
480  		kfree(s->last_msg);
481  		kfree(s);
482  	}
483  
484  	mutex_unlock(&c->fsck_error_msgs_lock);
485  }
486