1  /* SPDX-License-Identifier: GPL-2.0 */
2  /*
3   * Portions Copyright (C) 1992 Drew Eckhardt
4   */
5  #ifndef _LINUX_BLKDEV_H
6  #define _LINUX_BLKDEV_H
7  
8  #include <linux/types.h>
9  #include <linux/blk_types.h>
10  #include <linux/device.h>
11  #include <linux/list.h>
12  #include <linux/llist.h>
13  #include <linux/minmax.h>
14  #include <linux/timer.h>
15  #include <linux/workqueue.h>
16  #include <linux/wait.h>
17  #include <linux/bio.h>
18  #include <linux/gfp.h>
19  #include <linux/kdev_t.h>
20  #include <linux/rcupdate.h>
21  #include <linux/percpu-refcount.h>
22  #include <linux/blkzoned.h>
23  #include <linux/sched.h>
24  #include <linux/sbitmap.h>
25  #include <linux/uuid.h>
26  #include <linux/xarray.h>
27  #include <linux/file.h>
28  
29  struct module;
30  struct request_queue;
31  struct elevator_queue;
32  struct blk_trace;
33  struct request;
34  struct sg_io_hdr;
35  struct blkcg_gq;
36  struct blk_flush_queue;
37  struct kiocb;
38  struct pr_ops;
39  struct rq_qos;
40  struct blk_queue_stats;
41  struct blk_stat_callback;
42  struct blk_crypto_profile;
43  
44  extern const struct device_type disk_type;
45  extern const struct device_type part_type;
46  extern const struct class block_class;
47  
48  /*
49   * Maximum number of blkcg policies allowed to be registered concurrently.
50   * Defined here to simplify include dependency.
51   */
52  #define BLKCG_MAX_POLS		6
53  
54  #define DISK_MAX_PARTS			256
55  #define DISK_NAME_LEN			32
56  
57  #define PARTITION_META_INFO_VOLNAMELTH	64
58  /*
59   * Enough for the string representation of any kind of UUID plus NULL.
60   * EFI UUID is 36 characters. MSDOS UUID is 11 characters.
61   */
62  #define PARTITION_META_INFO_UUIDLTH	(UUID_STRING_LEN + 1)
63  
64  struct partition_meta_info {
65  	char uuid[PARTITION_META_INFO_UUIDLTH];
66  	u8 volname[PARTITION_META_INFO_VOLNAMELTH];
67  };
68  
69  /**
70   * DOC: genhd capability flags
71   *
72   * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to
73   * removable media.  When set, the device remains present even when media is not
74   * inserted.  Shall not be set for devices which are removed entirely when the
75   * media is removed.
76   *
77   * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events,
78   * doesn't appear in sysfs, and can't be opened from userspace or using
79   * blkdev_get*. Used for the underlying components of multipath devices.
80   *
81   * ``GENHD_FL_NO_PART``: partition support is disabled.  The kernel will not
82   * scan for partitions from add_disk, and users can't add partitions manually.
83   *
84   */
85  enum {
86  	GENHD_FL_REMOVABLE			= 1 << 0,
87  	GENHD_FL_HIDDEN				= 1 << 1,
88  	GENHD_FL_NO_PART			= 1 << 2,
89  };
90  
91  enum {
92  	DISK_EVENT_MEDIA_CHANGE			= 1 << 0, /* media changed */
93  	DISK_EVENT_EJECT_REQUEST		= 1 << 1, /* eject requested */
94  };
95  
96  enum {
97  	/* Poll even if events_poll_msecs is unset */
98  	DISK_EVENT_FLAG_POLL			= 1 << 0,
99  	/* Forward events to udev */
100  	DISK_EVENT_FLAG_UEVENT			= 1 << 1,
101  	/* Block event polling when open for exclusive write */
102  	DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE	= 1 << 2,
103  };
104  
105  struct disk_events;
106  struct badblocks;
107  
108  enum blk_integrity_checksum {
109  	BLK_INTEGRITY_CSUM_NONE		= 0,
110  	BLK_INTEGRITY_CSUM_IP		= 1,
111  	BLK_INTEGRITY_CSUM_CRC		= 2,
112  	BLK_INTEGRITY_CSUM_CRC64	= 3,
113  } __packed ;
114  
115  struct blk_integrity {
116  	unsigned char				flags;
117  	enum blk_integrity_checksum		csum_type;
118  	unsigned char				tuple_size;
119  	unsigned char				pi_offset;
120  	unsigned char				interval_exp;
121  	unsigned char				tag_size;
122  };
123  
124  typedef unsigned int __bitwise blk_mode_t;
125  
126  /* open for reading */
127  #define BLK_OPEN_READ		((__force blk_mode_t)(1 << 0))
128  /* open for writing */
129  #define BLK_OPEN_WRITE		((__force blk_mode_t)(1 << 1))
130  /* open exclusively (vs other exclusive openers */
131  #define BLK_OPEN_EXCL		((__force blk_mode_t)(1 << 2))
132  /* opened with O_NDELAY */
133  #define BLK_OPEN_NDELAY		((__force blk_mode_t)(1 << 3))
134  /* open for "writes" only for ioctls (specialy hack for floppy.c) */
135  #define BLK_OPEN_WRITE_IOCTL	((__force blk_mode_t)(1 << 4))
136  /* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */
137  #define BLK_OPEN_RESTRICT_WRITES	((__force blk_mode_t)(1 << 5))
138  /* return partition scanning errors */
139  #define BLK_OPEN_STRICT_SCAN	((__force blk_mode_t)(1 << 6))
140  
141  struct gendisk {
142  	/*
143  	 * major/first_minor/minors should not be set by any new driver, the
144  	 * block core will take care of allocating them automatically.
145  	 */
146  	int major;
147  	int first_minor;
148  	int minors;
149  
150  	char disk_name[DISK_NAME_LEN];	/* name of major driver */
151  
152  	unsigned short events;		/* supported events */
153  	unsigned short event_flags;	/* flags related to event processing */
154  
155  	struct xarray part_tbl;
156  	struct block_device *part0;
157  
158  	const struct block_device_operations *fops;
159  	struct request_queue *queue;
160  	void *private_data;
161  
162  	struct bio_set bio_split;
163  
164  	int flags;
165  	unsigned long state;
166  #define GD_NEED_PART_SCAN		0
167  #define GD_READ_ONLY			1
168  #define GD_DEAD				2
169  #define GD_NATIVE_CAPACITY		3
170  #define GD_ADDED			4
171  #define GD_SUPPRESS_PART_SCAN		5
172  #define GD_OWNS_QUEUE			6
173  
174  	struct mutex open_mutex;	/* open/close mutex */
175  	unsigned open_partitions;	/* number of open partitions */
176  
177  	struct backing_dev_info	*bdi;
178  	struct kobject queue_kobj;	/* the queue/ directory */
179  	struct kobject *slave_dir;
180  #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
181  	struct list_head slave_bdevs;
182  #endif
183  	struct timer_rand_state *random;
184  	atomic_t sync_io;		/* RAID */
185  	struct disk_events *ev;
186  
187  #ifdef CONFIG_BLK_DEV_ZONED
188  	/*
189  	 * Zoned block device information. Reads of this information must be
190  	 * protected with blk_queue_enter() / blk_queue_exit(). Modifying this
191  	 * information is only allowed while no requests are being processed.
192  	 * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue().
193  	 */
194  	unsigned int		nr_zones;
195  	unsigned int		zone_capacity;
196  	unsigned int		last_zone_capacity;
197  	unsigned long		*conv_zones_bitmap;
198  	unsigned int            zone_wplugs_hash_bits;
199  	spinlock_t              zone_wplugs_lock;
200  	struct mempool_s	*zone_wplugs_pool;
201  	struct hlist_head       *zone_wplugs_hash;
202  	struct list_head        zone_wplugs_err_list;
203  	struct work_struct	zone_wplugs_work;
204  	struct workqueue_struct *zone_wplugs_wq;
205  #endif /* CONFIG_BLK_DEV_ZONED */
206  
207  #if IS_ENABLED(CONFIG_CDROM)
208  	struct cdrom_device_info *cdi;
209  #endif
210  	int node_id;
211  	struct badblocks *bb;
212  	struct lockdep_map lockdep_map;
213  	u64 diskseq;
214  	blk_mode_t open_mode;
215  
216  	/*
217  	 * Independent sector access ranges. This is always NULL for
218  	 * devices that do not have multiple independent access ranges.
219  	 */
220  	struct blk_independent_access_ranges *ia_ranges;
221  };
222  
223  /**
224   * disk_openers - returns how many openers are there for a disk
225   * @disk: disk to check
226   *
227   * This returns the number of openers for a disk.  Note that this value is only
228   * stable if disk->open_mutex is held.
229   *
230   * Note: Due to a quirk in the block layer open code, each open partition is
231   * only counted once even if there are multiple openers.
232   */
disk_openers(struct gendisk * disk)233  static inline unsigned int disk_openers(struct gendisk *disk)
234  {
235  	return atomic_read(&disk->part0->bd_openers);
236  }
237  
238  /**
239   * disk_has_partscan - return %true if partition scanning is enabled on a disk
240   * @disk: disk to check
241   *
242   * Returns %true if partitions scanning is enabled for @disk, or %false if
243   * partition scanning is disabled either permanently or temporarily.
244   */
disk_has_partscan(struct gendisk * disk)245  static inline bool disk_has_partscan(struct gendisk *disk)
246  {
247  	return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) &&
248  		!test_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
249  }
250  
251  /*
252   * The gendisk is refcounted by the part0 block_device, and the bd_device
253   * therein is also used for device model presentation in sysfs.
254   */
255  #define dev_to_disk(device) \
256  	(dev_to_bdev(device)->bd_disk)
257  #define disk_to_dev(disk) \
258  	(&((disk)->part0->bd_device))
259  
260  #if IS_REACHABLE(CONFIG_CDROM)
261  #define disk_to_cdi(disk)	((disk)->cdi)
262  #else
263  #define disk_to_cdi(disk)	NULL
264  #endif
265  
disk_devt(struct gendisk * disk)266  static inline dev_t disk_devt(struct gendisk *disk)
267  {
268  	return MKDEV(disk->major, disk->first_minor);
269  }
270  
271  /* blk_validate_limits() validates bsize, so drivers don't usually need to */
blk_validate_block_size(unsigned long bsize)272  static inline int blk_validate_block_size(unsigned long bsize)
273  {
274  	if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize))
275  		return -EINVAL;
276  
277  	return 0;
278  }
279  
blk_op_is_passthrough(blk_opf_t op)280  static inline bool blk_op_is_passthrough(blk_opf_t op)
281  {
282  	op &= REQ_OP_MASK;
283  	return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
284  }
285  
286  /* flags set by the driver in queue_limits.features */
287  typedef unsigned int __bitwise blk_features_t;
288  
289  /* supports a volatile write cache */
290  #define BLK_FEAT_WRITE_CACHE		((__force blk_features_t)(1u << 0))
291  
292  /* supports passing on the FUA bit */
293  #define BLK_FEAT_FUA			((__force blk_features_t)(1u << 1))
294  
295  /* rotational device (hard drive or floppy) */
296  #define BLK_FEAT_ROTATIONAL		((__force blk_features_t)(1u << 2))
297  
298  /* contributes to the random number pool */
299  #define BLK_FEAT_ADD_RANDOM		((__force blk_features_t)(1u << 3))
300  
301  /* do disk/partitions IO accounting */
302  #define BLK_FEAT_IO_STAT		((__force blk_features_t)(1u << 4))
303  
304  /* don't modify data until writeback is done */
305  #define BLK_FEAT_STABLE_WRITES		((__force blk_features_t)(1u << 5))
306  
307  /* always completes in submit context */
308  #define BLK_FEAT_SYNCHRONOUS		((__force blk_features_t)(1u << 6))
309  
310  /* supports REQ_NOWAIT */
311  #define BLK_FEAT_NOWAIT			((__force blk_features_t)(1u << 7))
312  
313  /* supports DAX */
314  #define BLK_FEAT_DAX			((__force blk_features_t)(1u << 8))
315  
316  /* supports I/O polling */
317  #define BLK_FEAT_POLL			((__force blk_features_t)(1u << 9))
318  
319  /* is a zoned device */
320  #define BLK_FEAT_ZONED			((__force blk_features_t)(1u << 10))
321  
322  /* supports PCI(e) p2p requests */
323  #define BLK_FEAT_PCI_P2PDMA		((__force blk_features_t)(1u << 12))
324  
325  /* skip this queue in blk_mq_(un)quiesce_tagset */
326  #define BLK_FEAT_SKIP_TAGSET_QUIESCE	((__force blk_features_t)(1u << 13))
327  
328  /* bounce all highmem pages */
329  #define BLK_FEAT_BOUNCE_HIGH		((__force blk_features_t)(1u << 14))
330  
331  /* undocumented magic for bcache */
332  #define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \
333  	((__force blk_features_t)(1u << 15))
334  
335  /*
336   * Flags automatically inherited when stacking limits.
337   */
338  #define BLK_FEAT_INHERIT_MASK \
339  	(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \
340  	 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | BLK_FEAT_BOUNCE_HIGH | \
341  	 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE)
342  
343  /* internal flags in queue_limits.flags */
344  typedef unsigned int __bitwise blk_flags_t;
345  
346  /* do not send FLUSH/FUA commands despite advertising a write cache */
347  #define BLK_FLAG_WRITE_CACHE_DISABLED	((__force blk_flags_t)(1u << 0))
348  
349  /* I/O topology is misaligned */
350  #define BLK_FLAG_MISALIGNED		((__force blk_flags_t)(1u << 1))
351  
352  struct queue_limits {
353  	blk_features_t		features;
354  	blk_flags_t		flags;
355  	unsigned long		seg_boundary_mask;
356  	unsigned long		virt_boundary_mask;
357  
358  	unsigned int		max_hw_sectors;
359  	unsigned int		max_dev_sectors;
360  	unsigned int		chunk_sectors;
361  	unsigned int		max_sectors;
362  	unsigned int		max_user_sectors;
363  	unsigned int		max_segment_size;
364  	unsigned int		physical_block_size;
365  	unsigned int		logical_block_size;
366  	unsigned int		alignment_offset;
367  	unsigned int		io_min;
368  	unsigned int		io_opt;
369  	unsigned int		max_discard_sectors;
370  	unsigned int		max_hw_discard_sectors;
371  	unsigned int		max_user_discard_sectors;
372  	unsigned int		max_secure_erase_sectors;
373  	unsigned int		max_write_zeroes_sectors;
374  	unsigned int		max_zone_append_sectors;
375  	unsigned int		discard_granularity;
376  	unsigned int		discard_alignment;
377  	unsigned int		zone_write_granularity;
378  
379  	/* atomic write limits */
380  	unsigned int		atomic_write_hw_max;
381  	unsigned int		atomic_write_max_sectors;
382  	unsigned int		atomic_write_hw_boundary;
383  	unsigned int		atomic_write_boundary_sectors;
384  	unsigned int		atomic_write_hw_unit_min;
385  	unsigned int		atomic_write_unit_min;
386  	unsigned int		atomic_write_hw_unit_max;
387  	unsigned int		atomic_write_unit_max;
388  
389  	unsigned short		max_segments;
390  	unsigned short		max_integrity_segments;
391  	unsigned short		max_discard_segments;
392  
393  	unsigned int		max_open_zones;
394  	unsigned int		max_active_zones;
395  
396  	/*
397  	 * Drivers that set dma_alignment to less than 511 must be prepared to
398  	 * handle individual bvec's that are not a multiple of a SECTOR_SIZE
399  	 * due to possible offsets.
400  	 */
401  	unsigned int		dma_alignment;
402  	unsigned int		dma_pad_mask;
403  
404  	struct blk_integrity	integrity;
405  };
406  
407  typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
408  			       void *data);
409  
410  #define BLK_ALL_ZONES  ((unsigned int)-1)
411  int blkdev_report_zones(struct block_device *bdev, sector_t sector,
412  		unsigned int nr_zones, report_zones_cb cb, void *data);
413  int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
414  		sector_t sectors, sector_t nr_sectors);
415  int blk_revalidate_disk_zones(struct gendisk *disk);
416  
417  /*
418   * Independent access ranges: struct blk_independent_access_range describes
419   * a range of contiguous sectors that can be accessed using device command
420   * execution resources that are independent from the resources used for
421   * other access ranges. This is typically found with single-LUN multi-actuator
422   * HDDs where each access range is served by a different set of heads.
423   * The set of independent ranges supported by the device is defined using
424   * struct blk_independent_access_ranges. The independent ranges must not overlap
425   * and must include all sectors within the disk capacity (no sector holes
426   * allowed).
427   * For a device with multiple ranges, requests targeting sectors in different
428   * ranges can be executed in parallel. A request can straddle an access range
429   * boundary.
430   */
431  struct blk_independent_access_range {
432  	struct kobject		kobj;
433  	sector_t		sector;
434  	sector_t		nr_sectors;
435  };
436  
437  struct blk_independent_access_ranges {
438  	struct kobject				kobj;
439  	bool					sysfs_registered;
440  	unsigned int				nr_ia_ranges;
441  	struct blk_independent_access_range	ia_range[];
442  };
443  
444  struct request_queue {
445  	/*
446  	 * The queue owner gets to use this for whatever they like.
447  	 * ll_rw_blk doesn't touch it.
448  	 */
449  	void			*queuedata;
450  
451  	struct elevator_queue	*elevator;
452  
453  	const struct blk_mq_ops	*mq_ops;
454  
455  	/* sw queues */
456  	struct blk_mq_ctx __percpu	*queue_ctx;
457  
458  	/*
459  	 * various queue flags, see QUEUE_* below
460  	 */
461  	unsigned long		queue_flags;
462  
463  	unsigned int		rq_timeout;
464  
465  	unsigned int		queue_depth;
466  
467  	refcount_t		refs;
468  
469  	/* hw dispatch queues */
470  	unsigned int		nr_hw_queues;
471  	struct xarray		hctx_table;
472  
473  	struct percpu_ref	q_usage_counter;
474  
475  	struct request		*last_merge;
476  
477  	spinlock_t		queue_lock;
478  
479  	int			quiesce_depth;
480  
481  	struct gendisk		*disk;
482  
483  	/*
484  	 * mq queue kobject
485  	 */
486  	struct kobject *mq_kobj;
487  
488  	struct queue_limits	limits;
489  
490  #ifdef CONFIG_PM
491  	struct device		*dev;
492  	enum rpm_status		rpm_status;
493  #endif
494  
495  	/*
496  	 * Number of contexts that have called blk_set_pm_only(). If this
497  	 * counter is above zero then only RQF_PM requests are processed.
498  	 */
499  	atomic_t		pm_only;
500  
501  	struct blk_queue_stats	*stats;
502  	struct rq_qos		*rq_qos;
503  	struct mutex		rq_qos_mutex;
504  
505  	/*
506  	 * ida allocated id for this queue.  Used to index queues from
507  	 * ioctx.
508  	 */
509  	int			id;
510  
511  	/*
512  	 * queue settings
513  	 */
514  	unsigned long		nr_requests;	/* Max # of requests */
515  
516  #ifdef CONFIG_BLK_INLINE_ENCRYPTION
517  	struct blk_crypto_profile *crypto_profile;
518  	struct kobject *crypto_kobject;
519  #endif
520  
521  	struct timer_list	timeout;
522  	struct work_struct	timeout_work;
523  
524  	atomic_t		nr_active_requests_shared_tags;
525  
526  	struct blk_mq_tags	*sched_shared_tags;
527  
528  	struct list_head	icq_list;
529  #ifdef CONFIG_BLK_CGROUP
530  	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
531  	struct blkcg_gq		*root_blkg;
532  	struct list_head	blkg_list;
533  	struct mutex		blkcg_mutex;
534  #endif
535  
536  	int			node;
537  
538  	spinlock_t		requeue_lock;
539  	struct list_head	requeue_list;
540  	struct delayed_work	requeue_work;
541  
542  #ifdef CONFIG_BLK_DEV_IO_TRACE
543  	struct blk_trace __rcu	*blk_trace;
544  #endif
545  	/*
546  	 * for flush operations
547  	 */
548  	struct blk_flush_queue	*fq;
549  	struct list_head	flush_list;
550  
551  	struct mutex		sysfs_lock;
552  	struct mutex		sysfs_dir_lock;
553  	struct mutex		limits_lock;
554  
555  	/*
556  	 * for reusing dead hctx instance in case of updating
557  	 * nr_hw_queues
558  	 */
559  	struct list_head	unused_hctx_list;
560  	spinlock_t		unused_hctx_lock;
561  
562  	int			mq_freeze_depth;
563  
564  #ifdef CONFIG_BLK_DEV_THROTTLING
565  	/* Throttle data */
566  	struct throtl_data *td;
567  #endif
568  	struct rcu_head		rcu_head;
569  	wait_queue_head_t	mq_freeze_wq;
570  	/*
571  	 * Protect concurrent access to q_usage_counter by
572  	 * percpu_ref_kill() and percpu_ref_reinit().
573  	 */
574  	struct mutex		mq_freeze_lock;
575  
576  	struct blk_mq_tag_set	*tag_set;
577  	struct list_head	tag_set_list;
578  
579  	struct dentry		*debugfs_dir;
580  	struct dentry		*sched_debugfs_dir;
581  	struct dentry		*rqos_debugfs_dir;
582  	/*
583  	 * Serializes all debugfs metadata operations using the above dentries.
584  	 */
585  	struct mutex		debugfs_mutex;
586  
587  	bool			mq_sysfs_init_done;
588  };
589  
590  /* Keep blk_queue_flag_name[] in sync with the definitions below */
591  enum {
592  	QUEUE_FLAG_DYING,		/* queue being torn down */
593  	QUEUE_FLAG_NOMERGES,		/* disable merge attempts */
594  	QUEUE_FLAG_SAME_COMP,		/* complete on same CPU-group */
595  	QUEUE_FLAG_FAIL_IO,		/* fake timeout */
596  	QUEUE_FLAG_NOXMERGES,		/* No extended merges */
597  	QUEUE_FLAG_SAME_FORCE,		/* force complete on same CPU */
598  	QUEUE_FLAG_INIT_DONE,		/* queue is initialized */
599  	QUEUE_FLAG_STATS,		/* track IO start and completion times */
600  	QUEUE_FLAG_REGISTERED,		/* queue has been registered to a disk */
601  	QUEUE_FLAG_QUIESCED,		/* queue has been quiesced */
602  	QUEUE_FLAG_RQ_ALLOC_TIME,	/* record rq->alloc_time_ns */
603  	QUEUE_FLAG_HCTX_ACTIVE,		/* at least one blk-mq hctx is active */
604  	QUEUE_FLAG_SQ_SCHED,		/* single queue style io dispatch */
605  	QUEUE_FLAG_MAX
606  };
607  
608  #define QUEUE_FLAG_MQ_DEFAULT	(1UL << QUEUE_FLAG_SAME_COMP)
609  
610  void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
611  void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
612  
613  #define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
614  #define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
615  #define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
616  #define blk_queue_noxmerges(q)	\
617  	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
618  #define blk_queue_nonrot(q)	(!((q)->limits.features & BLK_FEAT_ROTATIONAL))
619  #define blk_queue_io_stat(q)	((q)->limits.features & BLK_FEAT_IO_STAT)
620  #define blk_queue_dax(q)	((q)->limits.features & BLK_FEAT_DAX)
621  #define blk_queue_pci_p2pdma(q)	((q)->limits.features & BLK_FEAT_PCI_P2PDMA)
622  #ifdef CONFIG_BLK_RQ_ALLOC_TIME
623  #define blk_queue_rq_alloc_time(q)	\
624  	test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
625  #else
626  #define blk_queue_rq_alloc_time(q)	false
627  #endif
628  
629  #define blk_noretry_request(rq) \
630  	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
631  			     REQ_FAILFAST_DRIVER))
632  #define blk_queue_quiesced(q)	test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
633  #define blk_queue_pm_only(q)	atomic_read(&(q)->pm_only)
634  #define blk_queue_registered(q)	test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
635  #define blk_queue_sq_sched(q)	test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
636  #define blk_queue_skip_tagset_quiesce(q) \
637  	((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE)
638  
639  extern void blk_set_pm_only(struct request_queue *q);
640  extern void blk_clear_pm_only(struct request_queue *q);
641  
642  #define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)
643  
644  #define dma_map_bvec(dev, bv, dir, attrs) \
645  	dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
646  	(dir), (attrs))
647  
queue_is_mq(struct request_queue * q)648  static inline bool queue_is_mq(struct request_queue *q)
649  {
650  	return q->mq_ops;
651  }
652  
653  #ifdef CONFIG_PM
queue_rpm_status(struct request_queue * q)654  static inline enum rpm_status queue_rpm_status(struct request_queue *q)
655  {
656  	return q->rpm_status;
657  }
658  #else
queue_rpm_status(struct request_queue * q)659  static inline enum rpm_status queue_rpm_status(struct request_queue *q)
660  {
661  	return RPM_ACTIVE;
662  }
663  #endif
664  
blk_queue_is_zoned(struct request_queue * q)665  static inline bool blk_queue_is_zoned(struct request_queue *q)
666  {
667  	return IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
668  		(q->limits.features & BLK_FEAT_ZONED);
669  }
670  
671  #ifdef CONFIG_BLK_DEV_ZONED
disk_nr_zones(struct gendisk * disk)672  static inline unsigned int disk_nr_zones(struct gendisk *disk)
673  {
674  	return disk->nr_zones;
675  }
676  bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs);
677  #else /* CONFIG_BLK_DEV_ZONED */
disk_nr_zones(struct gendisk * disk)678  static inline unsigned int disk_nr_zones(struct gendisk *disk)
679  {
680  	return 0;
681  }
blk_zone_plug_bio(struct bio * bio,unsigned int nr_segs)682  static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs)
683  {
684  	return false;
685  }
686  #endif /* CONFIG_BLK_DEV_ZONED */
687  
disk_zone_no(struct gendisk * disk,sector_t sector)688  static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
689  {
690  	if (!blk_queue_is_zoned(disk->queue))
691  		return 0;
692  	return sector >> ilog2(disk->queue->limits.chunk_sectors);
693  }
694  
bdev_nr_zones(struct block_device * bdev)695  static inline unsigned int bdev_nr_zones(struct block_device *bdev)
696  {
697  	return disk_nr_zones(bdev->bd_disk);
698  }
699  
bdev_max_open_zones(struct block_device * bdev)700  static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
701  {
702  	return bdev->bd_disk->queue->limits.max_open_zones;
703  }
704  
bdev_max_active_zones(struct block_device * bdev)705  static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
706  {
707  	return bdev->bd_disk->queue->limits.max_active_zones;
708  }
709  
blk_queue_depth(struct request_queue * q)710  static inline unsigned int blk_queue_depth(struct request_queue *q)
711  {
712  	if (q->queue_depth)
713  		return q->queue_depth;
714  
715  	return q->nr_requests;
716  }
717  
718  /*
719   * default timeout for SG_IO if none specified
720   */
721  #define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
722  #define BLK_MIN_SG_TIMEOUT	(7 * HZ)
723  
724  /* This should not be used directly - use rq_for_each_segment */
725  #define for_each_bio(_bio)		\
726  	for (; _bio; _bio = _bio->bi_next)
727  
728  int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
729  				 const struct attribute_group **groups);
add_disk(struct gendisk * disk)730  static inline int __must_check add_disk(struct gendisk *disk)
731  {
732  	return device_add_disk(NULL, disk, NULL);
733  }
734  void del_gendisk(struct gendisk *gp);
735  void invalidate_disk(struct gendisk *disk);
736  void set_disk_ro(struct gendisk *disk, bool read_only);
737  void disk_uevent(struct gendisk *disk, enum kobject_action action);
738  
bdev_partno(const struct block_device * bdev)739  static inline u8 bdev_partno(const struct block_device *bdev)
740  {
741  	return atomic_read(&bdev->__bd_flags) & BD_PARTNO;
742  }
743  
bdev_test_flag(const struct block_device * bdev,unsigned flag)744  static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag)
745  {
746  	return atomic_read(&bdev->__bd_flags) & flag;
747  }
748  
bdev_set_flag(struct block_device * bdev,unsigned flag)749  static inline void bdev_set_flag(struct block_device *bdev, unsigned flag)
750  {
751  	atomic_or(flag, &bdev->__bd_flags);
752  }
753  
bdev_clear_flag(struct block_device * bdev,unsigned flag)754  static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag)
755  {
756  	atomic_andnot(flag, &bdev->__bd_flags);
757  }
758  
get_disk_ro(struct gendisk * disk)759  static inline int get_disk_ro(struct gendisk *disk)
760  {
761  	return bdev_test_flag(disk->part0, BD_READ_ONLY) ||
762  		test_bit(GD_READ_ONLY, &disk->state);
763  }
764  
bdev_read_only(struct block_device * bdev)765  static inline int bdev_read_only(struct block_device *bdev)
766  {
767  	return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk);
768  }
769  
770  bool set_capacity_and_notify(struct gendisk *disk, sector_t size);
771  void disk_force_media_change(struct gendisk *disk);
772  void bdev_mark_dead(struct block_device *bdev, bool surprise);
773  
774  void add_disk_randomness(struct gendisk *disk) __latent_entropy;
775  void rand_initialize_disk(struct gendisk *disk);
776  
get_start_sect(struct block_device * bdev)777  static inline sector_t get_start_sect(struct block_device *bdev)
778  {
779  	return bdev->bd_start_sect;
780  }
781  
bdev_nr_sectors(struct block_device * bdev)782  static inline sector_t bdev_nr_sectors(struct block_device *bdev)
783  {
784  	return bdev->bd_nr_sectors;
785  }
786  
bdev_nr_bytes(struct block_device * bdev)787  static inline loff_t bdev_nr_bytes(struct block_device *bdev)
788  {
789  	return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT;
790  }
791  
get_capacity(struct gendisk * disk)792  static inline sector_t get_capacity(struct gendisk *disk)
793  {
794  	return bdev_nr_sectors(disk->part0);
795  }
796  
sb_bdev_nr_blocks(struct super_block * sb)797  static inline u64 sb_bdev_nr_blocks(struct super_block *sb)
798  {
799  	return bdev_nr_sectors(sb->s_bdev) >>
800  		(sb->s_blocksize_bits - SECTOR_SHIFT);
801  }
802  
803  int bdev_disk_changed(struct gendisk *disk, bool invalidate);
804  
805  void put_disk(struct gendisk *disk);
806  struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node,
807  		struct lock_class_key *lkclass);
808  
809  /**
810   * blk_alloc_disk - allocate a gendisk structure
811   * @lim: queue limits to be used for this disk.
812   * @node_id: numa node to allocate on
813   *
814   * Allocate and pre-initialize a gendisk structure for use with BIO based
815   * drivers.
816   *
817   * Returns an ERR_PTR on error, else the allocated disk.
818   *
819   * Context: can sleep
820   */
821  #define blk_alloc_disk(lim, node_id)					\
822  ({									\
823  	static struct lock_class_key __key;				\
824  									\
825  	__blk_alloc_disk(lim, node_id, &__key);				\
826  })
827  
828  int __register_blkdev(unsigned int major, const char *name,
829  		void (*probe)(dev_t devt));
830  #define register_blkdev(major, name) \
831  	__register_blkdev(major, name, NULL)
832  void unregister_blkdev(unsigned int major, const char *name);
833  
834  bool disk_check_media_change(struct gendisk *disk);
835  void set_capacity(struct gendisk *disk, sector_t size);
836  
837  #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
838  int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk);
839  void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk);
840  #else
bd_link_disk_holder(struct block_device * bdev,struct gendisk * disk)841  static inline int bd_link_disk_holder(struct block_device *bdev,
842  				      struct gendisk *disk)
843  {
844  	return 0;
845  }
bd_unlink_disk_holder(struct block_device * bdev,struct gendisk * disk)846  static inline void bd_unlink_disk_holder(struct block_device *bdev,
847  					 struct gendisk *disk)
848  {
849  }
850  #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */
851  
852  dev_t part_devt(struct gendisk *disk, u8 partno);
853  void inc_diskseq(struct gendisk *disk);
854  void blk_request_module(dev_t devt);
855  
856  extern int blk_register_queue(struct gendisk *disk);
857  extern void blk_unregister_queue(struct gendisk *disk);
858  void submit_bio_noacct(struct bio *bio);
859  struct bio *bio_split_to_limits(struct bio *bio);
860  
861  extern int blk_lld_busy(struct request_queue *q);
862  extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
863  extern void blk_queue_exit(struct request_queue *q);
864  extern void blk_sync_queue(struct request_queue *q);
865  
866  /* Helper to convert REQ_OP_XXX to its string format XXX */
867  extern const char *blk_op_str(enum req_op op);
868  
869  int blk_status_to_errno(blk_status_t status);
870  blk_status_t errno_to_blk_status(int errno);
871  const char *blk_status_to_str(blk_status_t status);
872  
873  /* only poll the hardware once, don't continue until a completion was found */
874  #define BLK_POLL_ONESHOT		(1 << 0)
875  int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
876  int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
877  			unsigned int flags);
878  
bdev_get_queue(struct block_device * bdev)879  static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
880  {
881  	return bdev->bd_queue;	/* this is never NULL */
882  }
883  
884  /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */
885  const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
886  
bio_zone_no(struct bio * bio)887  static inline unsigned int bio_zone_no(struct bio *bio)
888  {
889  	return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
890  }
891  
bio_straddles_zones(struct bio * bio)892  static inline bool bio_straddles_zones(struct bio *bio)
893  {
894  	return bio_sectors(bio) &&
895  		bio_zone_no(bio) !=
896  		disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1);
897  }
898  
899  /*
900   * Return how much within the boundary is left to be used for I/O at a given
901   * offset.
902   */
blk_boundary_sectors_left(sector_t offset,unsigned int boundary_sectors)903  static inline unsigned int blk_boundary_sectors_left(sector_t offset,
904  		unsigned int boundary_sectors)
905  {
906  	if (unlikely(!is_power_of_2(boundary_sectors)))
907  		return boundary_sectors - sector_div(offset, boundary_sectors);
908  	return boundary_sectors - (offset & (boundary_sectors - 1));
909  }
910  
911  /**
912   * queue_limits_start_update - start an atomic update of queue limits
913   * @q:		queue to update
914   *
915   * This functions starts an atomic update of the queue limits.  It takes a lock
916   * to prevent other updates and returns a snapshot of the current limits that
917   * the caller can modify.  The caller must call queue_limits_commit_update()
918   * to finish the update.
919   *
920   * Context: process context.  The caller must have frozen the queue or ensured
921   * that there is outstanding I/O by other means.
922   */
923  static inline struct queue_limits
queue_limits_start_update(struct request_queue * q)924  queue_limits_start_update(struct request_queue *q)
925  {
926  	mutex_lock(&q->limits_lock);
927  	return q->limits;
928  }
929  int queue_limits_commit_update(struct request_queue *q,
930  		struct queue_limits *lim);
931  int queue_limits_set(struct request_queue *q, struct queue_limits *lim);
932  
933  /**
934   * queue_limits_cancel_update - cancel an atomic update of queue limits
935   * @q:		queue to update
936   *
937   * This functions cancels an atomic update of the queue limits started by
938   * queue_limits_start_update() and should be used when an error occurs after
939   * starting update.
940   */
queue_limits_cancel_update(struct request_queue * q)941  static inline void queue_limits_cancel_update(struct request_queue *q)
942  {
943  	mutex_unlock(&q->limits_lock);
944  }
945  
946  /*
947   * These helpers are for drivers that have sloppy feature negotiation and might
948   * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O
949   * completion handler when the device returned an indicator that the respective
950   * feature is not actually supported.  They are racy and the driver needs to
951   * cope with that.  Try to avoid this scheme if you can.
952   */
blk_queue_disable_discard(struct request_queue * q)953  static inline void blk_queue_disable_discard(struct request_queue *q)
954  {
955  	q->limits.max_discard_sectors = 0;
956  }
957  
blk_queue_disable_secure_erase(struct request_queue * q)958  static inline void blk_queue_disable_secure_erase(struct request_queue *q)
959  {
960  	q->limits.max_secure_erase_sectors = 0;
961  }
962  
blk_queue_disable_write_zeroes(struct request_queue * q)963  static inline void blk_queue_disable_write_zeroes(struct request_queue *q)
964  {
965  	q->limits.max_write_zeroes_sectors = 0;
966  }
967  
968  /*
969   * Access functions for manipulating queue properties
970   */
971  extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
972  extern void blk_set_stacking_limits(struct queue_limits *lim);
973  extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
974  			    sector_t offset);
975  void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev,
976  		sector_t offset, const char *pfx);
977  extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
978  
979  struct blk_independent_access_ranges *
980  disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges);
981  void disk_set_independent_access_ranges(struct gendisk *disk,
982  				struct blk_independent_access_ranges *iars);
983  
984  bool __must_check blk_get_queue(struct request_queue *);
985  extern void blk_put_queue(struct request_queue *);
986  
987  void blk_mark_disk_dead(struct gendisk *disk);
988  
989  #ifdef CONFIG_BLOCK
990  /*
991   * blk_plug permits building a queue of related requests by holding the I/O
992   * fragments for a short period. This allows merging of sequential requests
993   * into single larger request. As the requests are moved from a per-task list to
994   * the device's request_queue in a batch, this results in improved scalability
995   * as the lock contention for request_queue lock is reduced.
996   *
997   * It is ok not to disable preemption when adding the request to the plug list
998   * or when attempting a merge. For details, please see schedule() where
999   * blk_flush_plug() is called.
1000   */
1001  struct blk_plug {
1002  	struct request *mq_list; /* blk-mq requests */
1003  
1004  	/* if ios_left is > 1, we can batch tag/rq allocations */
1005  	struct request *cached_rq;
1006  	u64 cur_ktime;
1007  	unsigned short nr_ios;
1008  
1009  	unsigned short rq_count;
1010  
1011  	bool multiple_queues;
1012  	bool has_elevator;
1013  
1014  	struct list_head cb_list; /* md requires an unplug callback */
1015  };
1016  
1017  struct blk_plug_cb;
1018  typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1019  struct blk_plug_cb {
1020  	struct list_head list;
1021  	blk_plug_cb_fn callback;
1022  	void *data;
1023  };
1024  extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1025  					     void *data, int size);
1026  extern void blk_start_plug(struct blk_plug *);
1027  extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short);
1028  extern void blk_finish_plug(struct blk_plug *);
1029  
1030  void __blk_flush_plug(struct blk_plug *plug, bool from_schedule);
blk_flush_plug(struct blk_plug * plug,bool async)1031  static inline void blk_flush_plug(struct blk_plug *plug, bool async)
1032  {
1033  	if (plug)
1034  		__blk_flush_plug(plug, async);
1035  }
1036  
1037  /*
1038   * tsk == current here
1039   */
blk_plug_invalidate_ts(struct task_struct * tsk)1040  static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
1041  {
1042  	struct blk_plug *plug = tsk->plug;
1043  
1044  	if (plug)
1045  		plug->cur_ktime = 0;
1046  	current->flags &= ~PF_BLOCK_TS;
1047  }
1048  
1049  int blkdev_issue_flush(struct block_device *bdev);
1050  long nr_blockdev_pages(void);
1051  #else /* CONFIG_BLOCK */
1052  struct blk_plug {
1053  };
1054  
blk_start_plug_nr_ios(struct blk_plug * plug,unsigned short nr_ios)1055  static inline void blk_start_plug_nr_ios(struct blk_plug *plug,
1056  					 unsigned short nr_ios)
1057  {
1058  }
1059  
blk_start_plug(struct blk_plug * plug)1060  static inline void blk_start_plug(struct blk_plug *plug)
1061  {
1062  }
1063  
blk_finish_plug(struct blk_plug * plug)1064  static inline void blk_finish_plug(struct blk_plug *plug)
1065  {
1066  }
1067  
blk_flush_plug(struct blk_plug * plug,bool async)1068  static inline void blk_flush_plug(struct blk_plug *plug, bool async)
1069  {
1070  }
1071  
blk_plug_invalidate_ts(struct task_struct * tsk)1072  static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
1073  {
1074  }
1075  
blkdev_issue_flush(struct block_device * bdev)1076  static inline int blkdev_issue_flush(struct block_device *bdev)
1077  {
1078  	return 0;
1079  }
1080  
nr_blockdev_pages(void)1081  static inline long nr_blockdev_pages(void)
1082  {
1083  	return 0;
1084  }
1085  #endif /* CONFIG_BLOCK */
1086  
1087  extern void blk_io_schedule(void);
1088  
1089  int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1090  		sector_t nr_sects, gfp_t gfp_mask);
1091  int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1092  		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop);
1093  int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
1094  		sector_t nr_sects, gfp_t gfp);
1095  
1096  #define BLKDEV_ZERO_NOUNMAP	(1 << 0)  /* do not free blocks */
1097  #define BLKDEV_ZERO_NOFALLBACK	(1 << 1)  /* don't write explicit zeroes */
1098  #define BLKDEV_ZERO_KILLABLE	(1 << 2)  /* interruptible by fatal signals */
1099  
1100  extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1101  		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1102  		unsigned flags);
1103  extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1104  		sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1105  
sb_issue_discard(struct super_block * sb,sector_t block,sector_t nr_blocks,gfp_t gfp_mask,unsigned long flags)1106  static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1107  		sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1108  {
1109  	return blkdev_issue_discard(sb->s_bdev,
1110  				    block << (sb->s_blocksize_bits -
1111  					      SECTOR_SHIFT),
1112  				    nr_blocks << (sb->s_blocksize_bits -
1113  						  SECTOR_SHIFT),
1114  				    gfp_mask);
1115  }
sb_issue_zeroout(struct super_block * sb,sector_t block,sector_t nr_blocks,gfp_t gfp_mask)1116  static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1117  		sector_t nr_blocks, gfp_t gfp_mask)
1118  {
1119  	return blkdev_issue_zeroout(sb->s_bdev,
1120  				    block << (sb->s_blocksize_bits -
1121  					      SECTOR_SHIFT),
1122  				    nr_blocks << (sb->s_blocksize_bits -
1123  						  SECTOR_SHIFT),
1124  				    gfp_mask, 0);
1125  }
1126  
bdev_is_partition(struct block_device * bdev)1127  static inline bool bdev_is_partition(struct block_device *bdev)
1128  {
1129  	return bdev_partno(bdev) != 0;
1130  }
1131  
1132  enum blk_default_limits {
1133  	BLK_MAX_SEGMENTS	= 128,
1134  	BLK_SAFE_MAX_SECTORS	= 255,
1135  	BLK_MAX_SEGMENT_SIZE	= 65536,
1136  	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
1137  };
1138  
1139  /*
1140   * Default upper limit for the software max_sectors limit used for
1141   * regular file system I/O.  This can be increased through sysfs.
1142   *
1143   * Not to be confused with the max_hw_sector limit that is entirely
1144   * controlled by the driver, usually based on hardware limits.
1145   */
1146  #define BLK_DEF_MAX_SECTORS_CAP	2560u
1147  
queue_segment_boundary(const struct request_queue * q)1148  static inline unsigned long queue_segment_boundary(const struct request_queue *q)
1149  {
1150  	return q->limits.seg_boundary_mask;
1151  }
1152  
queue_virt_boundary(const struct request_queue * q)1153  static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1154  {
1155  	return q->limits.virt_boundary_mask;
1156  }
1157  
queue_max_sectors(const struct request_queue * q)1158  static inline unsigned int queue_max_sectors(const struct request_queue *q)
1159  {
1160  	return q->limits.max_sectors;
1161  }
1162  
queue_max_bytes(struct request_queue * q)1163  static inline unsigned int queue_max_bytes(struct request_queue *q)
1164  {
1165  	return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9;
1166  }
1167  
queue_max_hw_sectors(const struct request_queue * q)1168  static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1169  {
1170  	return q->limits.max_hw_sectors;
1171  }
1172  
queue_max_segments(const struct request_queue * q)1173  static inline unsigned short queue_max_segments(const struct request_queue *q)
1174  {
1175  	return q->limits.max_segments;
1176  }
1177  
queue_max_discard_segments(const struct request_queue * q)1178  static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1179  {
1180  	return q->limits.max_discard_segments;
1181  }
1182  
queue_max_segment_size(const struct request_queue * q)1183  static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1184  {
1185  	return q->limits.max_segment_size;
1186  }
1187  
1188  static inline unsigned int
queue_limits_max_zone_append_sectors(const struct queue_limits * l)1189  queue_limits_max_zone_append_sectors(const struct queue_limits *l)
1190  {
1191  	unsigned int max_sectors = min(l->chunk_sectors, l->max_hw_sectors);
1192  
1193  	return min_not_zero(l->max_zone_append_sectors, max_sectors);
1194  }
1195  
queue_max_zone_append_sectors(struct request_queue * q)1196  static inline unsigned int queue_max_zone_append_sectors(struct request_queue *q)
1197  {
1198  	if (!blk_queue_is_zoned(q))
1199  		return 0;
1200  
1201  	return queue_limits_max_zone_append_sectors(&q->limits);
1202  }
1203  
queue_emulates_zone_append(struct request_queue * q)1204  static inline bool queue_emulates_zone_append(struct request_queue *q)
1205  {
1206  	return blk_queue_is_zoned(q) && !q->limits.max_zone_append_sectors;
1207  }
1208  
bdev_emulates_zone_append(struct block_device * bdev)1209  static inline bool bdev_emulates_zone_append(struct block_device *bdev)
1210  {
1211  	return queue_emulates_zone_append(bdev_get_queue(bdev));
1212  }
1213  
1214  static inline unsigned int
bdev_max_zone_append_sectors(struct block_device * bdev)1215  bdev_max_zone_append_sectors(struct block_device *bdev)
1216  {
1217  	return queue_max_zone_append_sectors(bdev_get_queue(bdev));
1218  }
1219  
bdev_max_segments(struct block_device * bdev)1220  static inline unsigned int bdev_max_segments(struct block_device *bdev)
1221  {
1222  	return queue_max_segments(bdev_get_queue(bdev));
1223  }
1224  
queue_logical_block_size(const struct request_queue * q)1225  static inline unsigned queue_logical_block_size(const struct request_queue *q)
1226  {
1227  	return q->limits.logical_block_size;
1228  }
1229  
bdev_logical_block_size(struct block_device * bdev)1230  static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
1231  {
1232  	return queue_logical_block_size(bdev_get_queue(bdev));
1233  }
1234  
queue_physical_block_size(const struct request_queue * q)1235  static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1236  {
1237  	return q->limits.physical_block_size;
1238  }
1239  
bdev_physical_block_size(struct block_device * bdev)1240  static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1241  {
1242  	return queue_physical_block_size(bdev_get_queue(bdev));
1243  }
1244  
queue_io_min(const struct request_queue * q)1245  static inline unsigned int queue_io_min(const struct request_queue *q)
1246  {
1247  	return q->limits.io_min;
1248  }
1249  
bdev_io_min(struct block_device * bdev)1250  static inline int bdev_io_min(struct block_device *bdev)
1251  {
1252  	return queue_io_min(bdev_get_queue(bdev));
1253  }
1254  
queue_io_opt(const struct request_queue * q)1255  static inline unsigned int queue_io_opt(const struct request_queue *q)
1256  {
1257  	return q->limits.io_opt;
1258  }
1259  
bdev_io_opt(struct block_device * bdev)1260  static inline int bdev_io_opt(struct block_device *bdev)
1261  {
1262  	return queue_io_opt(bdev_get_queue(bdev));
1263  }
1264  
1265  static inline unsigned int
queue_zone_write_granularity(const struct request_queue * q)1266  queue_zone_write_granularity(const struct request_queue *q)
1267  {
1268  	return q->limits.zone_write_granularity;
1269  }
1270  
1271  static inline unsigned int
bdev_zone_write_granularity(struct block_device * bdev)1272  bdev_zone_write_granularity(struct block_device *bdev)
1273  {
1274  	return queue_zone_write_granularity(bdev_get_queue(bdev));
1275  }
1276  
1277  int bdev_alignment_offset(struct block_device *bdev);
1278  unsigned int bdev_discard_alignment(struct block_device *bdev);
1279  
bdev_max_discard_sectors(struct block_device * bdev)1280  static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev)
1281  {
1282  	return bdev_get_queue(bdev)->limits.max_discard_sectors;
1283  }
1284  
bdev_discard_granularity(struct block_device * bdev)1285  static inline unsigned int bdev_discard_granularity(struct block_device *bdev)
1286  {
1287  	return bdev_get_queue(bdev)->limits.discard_granularity;
1288  }
1289  
1290  static inline unsigned int
bdev_max_secure_erase_sectors(struct block_device * bdev)1291  bdev_max_secure_erase_sectors(struct block_device *bdev)
1292  {
1293  	return bdev_get_queue(bdev)->limits.max_secure_erase_sectors;
1294  }
1295  
bdev_write_zeroes_sectors(struct block_device * bdev)1296  static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1297  {
1298  	return bdev_get_queue(bdev)->limits.max_write_zeroes_sectors;
1299  }
1300  
bdev_nonrot(struct block_device * bdev)1301  static inline bool bdev_nonrot(struct block_device *bdev)
1302  {
1303  	return blk_queue_nonrot(bdev_get_queue(bdev));
1304  }
1305  
bdev_synchronous(struct block_device * bdev)1306  static inline bool bdev_synchronous(struct block_device *bdev)
1307  {
1308  	return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
1309  }
1310  
bdev_stable_writes(struct block_device * bdev)1311  static inline bool bdev_stable_writes(struct block_device *bdev)
1312  {
1313  	struct request_queue *q = bdev_get_queue(bdev);
1314  
1315  	if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1316  	    q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE)
1317  		return true;
1318  	return q->limits.features & BLK_FEAT_STABLE_WRITES;
1319  }
1320  
blk_queue_write_cache(struct request_queue * q)1321  static inline bool blk_queue_write_cache(struct request_queue *q)
1322  {
1323  	return (q->limits.features & BLK_FEAT_WRITE_CACHE) &&
1324  		!(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED);
1325  }
1326  
bdev_write_cache(struct block_device * bdev)1327  static inline bool bdev_write_cache(struct block_device *bdev)
1328  {
1329  	return blk_queue_write_cache(bdev_get_queue(bdev));
1330  }
1331  
bdev_fua(struct block_device * bdev)1332  static inline bool bdev_fua(struct block_device *bdev)
1333  {
1334  	return bdev_get_queue(bdev)->limits.features & BLK_FEAT_FUA;
1335  }
1336  
bdev_nowait(struct block_device * bdev)1337  static inline bool bdev_nowait(struct block_device *bdev)
1338  {
1339  	return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT;
1340  }
1341  
bdev_is_zoned(struct block_device * bdev)1342  static inline bool bdev_is_zoned(struct block_device *bdev)
1343  {
1344  	return blk_queue_is_zoned(bdev_get_queue(bdev));
1345  }
1346  
bdev_zone_no(struct block_device * bdev,sector_t sec)1347  static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec)
1348  {
1349  	return disk_zone_no(bdev->bd_disk, sec);
1350  }
1351  
bdev_zone_sectors(struct block_device * bdev)1352  static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1353  {
1354  	struct request_queue *q = bdev_get_queue(bdev);
1355  
1356  	if (!blk_queue_is_zoned(q))
1357  		return 0;
1358  	return q->limits.chunk_sectors;
1359  }
1360  
bdev_offset_from_zone_start(struct block_device * bdev,sector_t sector)1361  static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev,
1362  						   sector_t sector)
1363  {
1364  	return sector & (bdev_zone_sectors(bdev) - 1);
1365  }
1366  
bio_offset_from_zone_start(struct bio * bio)1367  static inline sector_t bio_offset_from_zone_start(struct bio *bio)
1368  {
1369  	return bdev_offset_from_zone_start(bio->bi_bdev,
1370  					   bio->bi_iter.bi_sector);
1371  }
1372  
bdev_is_zone_start(struct block_device * bdev,sector_t sector)1373  static inline bool bdev_is_zone_start(struct block_device *bdev,
1374  				      sector_t sector)
1375  {
1376  	return bdev_offset_from_zone_start(bdev, sector) == 0;
1377  }
1378  
queue_dma_alignment(const struct request_queue * q)1379  static inline int queue_dma_alignment(const struct request_queue *q)
1380  {
1381  	return q->limits.dma_alignment;
1382  }
1383  
1384  static inline unsigned int
queue_atomic_write_unit_max_bytes(const struct request_queue * q)1385  queue_atomic_write_unit_max_bytes(const struct request_queue *q)
1386  {
1387  	return q->limits.atomic_write_unit_max;
1388  }
1389  
1390  static inline unsigned int
queue_atomic_write_unit_min_bytes(const struct request_queue * q)1391  queue_atomic_write_unit_min_bytes(const struct request_queue *q)
1392  {
1393  	return q->limits.atomic_write_unit_min;
1394  }
1395  
1396  static inline unsigned int
queue_atomic_write_boundary_bytes(const struct request_queue * q)1397  queue_atomic_write_boundary_bytes(const struct request_queue *q)
1398  {
1399  	return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT;
1400  }
1401  
1402  static inline unsigned int
queue_atomic_write_max_bytes(const struct request_queue * q)1403  queue_atomic_write_max_bytes(const struct request_queue *q)
1404  {
1405  	return q->limits.atomic_write_max_sectors << SECTOR_SHIFT;
1406  }
1407  
bdev_dma_alignment(struct block_device * bdev)1408  static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
1409  {
1410  	return queue_dma_alignment(bdev_get_queue(bdev));
1411  }
1412  
bdev_iter_is_aligned(struct block_device * bdev,struct iov_iter * iter)1413  static inline bool bdev_iter_is_aligned(struct block_device *bdev,
1414  					struct iov_iter *iter)
1415  {
1416  	return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev),
1417  				   bdev_logical_block_size(bdev) - 1);
1418  }
1419  
blk_lim_dma_alignment_and_pad(struct queue_limits * lim)1420  static inline int blk_lim_dma_alignment_and_pad(struct queue_limits *lim)
1421  {
1422  	return lim->dma_alignment | lim->dma_pad_mask;
1423  }
1424  
blk_rq_aligned(struct request_queue * q,unsigned long addr,unsigned int len)1425  static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1426  				 unsigned int len)
1427  {
1428  	unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits);
1429  
1430  	return !(addr & alignment) && !(len & alignment);
1431  }
1432  
1433  /* assumes size > 256 */
blksize_bits(unsigned int size)1434  static inline unsigned int blksize_bits(unsigned int size)
1435  {
1436  	return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT;
1437  }
1438  
1439  int kblockd_schedule_work(struct work_struct *work);
1440  int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1441  
1442  #define MODULE_ALIAS_BLOCKDEV(major,minor) \
1443  	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1444  #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1445  	MODULE_ALIAS("block-major-" __stringify(major) "-*")
1446  
1447  #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1448  
1449  bool blk_crypto_register(struct blk_crypto_profile *profile,
1450  			 struct request_queue *q);
1451  
1452  #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1453  
blk_crypto_register(struct blk_crypto_profile * profile,struct request_queue * q)1454  static inline bool blk_crypto_register(struct blk_crypto_profile *profile,
1455  				       struct request_queue *q)
1456  {
1457  	return true;
1458  }
1459  
1460  #endif /* CONFIG_BLK_INLINE_ENCRYPTION */
1461  
1462  enum blk_unique_id {
1463  	/* these match the Designator Types specified in SPC */
1464  	BLK_UID_T10	= 1,
1465  	BLK_UID_EUI64	= 2,
1466  	BLK_UID_NAA	= 3,
1467  };
1468  
1469  struct block_device_operations {
1470  	void (*submit_bio)(struct bio *bio);
1471  	int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob,
1472  			unsigned int flags);
1473  	int (*open)(struct gendisk *disk, blk_mode_t mode);
1474  	void (*release)(struct gendisk *disk);
1475  	int (*ioctl)(struct block_device *bdev, blk_mode_t mode,
1476  			unsigned cmd, unsigned long arg);
1477  	int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode,
1478  			unsigned cmd, unsigned long arg);
1479  	unsigned int (*check_events) (struct gendisk *disk,
1480  				      unsigned int clearing);
1481  	void (*unlock_native_capacity) (struct gendisk *);
1482  	int (*getgeo)(struct block_device *, struct hd_geometry *);
1483  	int (*set_read_only)(struct block_device *bdev, bool ro);
1484  	void (*free_disk)(struct gendisk *disk);
1485  	/* this callback is with swap_lock and sometimes page table lock held */
1486  	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1487  	int (*report_zones)(struct gendisk *, sector_t sector,
1488  			unsigned int nr_zones, report_zones_cb cb, void *data);
1489  	char *(*devnode)(struct gendisk *disk, umode_t *mode);
1490  	/* returns the length of the identifier or a negative errno: */
1491  	int (*get_unique_id)(struct gendisk *disk, u8 id[16],
1492  			enum blk_unique_id id_type);
1493  	struct module *owner;
1494  	const struct pr_ops *pr_ops;
1495  
1496  	/*
1497  	 * Special callback for probing GPT entry at a given sector.
1498  	 * Needed by Android devices, used by GPT scanner and MMC blk
1499  	 * driver.
1500  	 */
1501  	int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector);
1502  };
1503  
1504  #ifdef CONFIG_COMPAT
1505  extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t,
1506  				      unsigned int, unsigned long);
1507  #else
1508  #define blkdev_compat_ptr_ioctl NULL
1509  #endif
1510  
blk_wake_io_task(struct task_struct * waiter)1511  static inline void blk_wake_io_task(struct task_struct *waiter)
1512  {
1513  	/*
1514  	 * If we're polling, the task itself is doing the completions. For
1515  	 * that case, we don't need to signal a wakeup, it's enough to just
1516  	 * mark us as RUNNING.
1517  	 */
1518  	if (waiter == current)
1519  		__set_current_state(TASK_RUNNING);
1520  	else
1521  		wake_up_process(waiter);
1522  }
1523  
1524  unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
1525  				 unsigned long start_time);
1526  void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
1527  		      unsigned int sectors, unsigned long start_time);
1528  
1529  unsigned long bio_start_io_acct(struct bio *bio);
1530  void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1531  		struct block_device *orig_bdev);
1532  
1533  /**
1534   * bio_end_io_acct - end I/O accounting for bio based drivers
1535   * @bio:	bio to end account for
1536   * @start_time:	start time returned by bio_start_io_acct()
1537   */
bio_end_io_acct(struct bio * bio,unsigned long start_time)1538  static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
1539  {
1540  	return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
1541  }
1542  
1543  int bdev_read_only(struct block_device *bdev);
1544  int set_blocksize(struct file *file, int size);
1545  
1546  int lookup_bdev(const char *pathname, dev_t *dev);
1547  
1548  void blkdev_show(struct seq_file *seqf, off_t offset);
1549  
1550  #define BDEVNAME_SIZE	32	/* Largest string for a blockdev identifier */
1551  #define BDEVT_SIZE	10	/* Largest string for MAJ:MIN for blkdev */
1552  #ifdef CONFIG_BLOCK
1553  #define BLKDEV_MAJOR_MAX	512
1554  #else
1555  #define BLKDEV_MAJOR_MAX	0
1556  #endif
1557  
1558  struct blk_holder_ops {
1559  	void (*mark_dead)(struct block_device *bdev, bool surprise);
1560  
1561  	/*
1562  	 * Sync the file system mounted on the block device.
1563  	 */
1564  	void (*sync)(struct block_device *bdev);
1565  
1566  	/*
1567  	 * Freeze the file system mounted on the block device.
1568  	 */
1569  	int (*freeze)(struct block_device *bdev);
1570  
1571  	/*
1572  	 * Thaw the file system mounted on the block device.
1573  	 */
1574  	int (*thaw)(struct block_device *bdev);
1575  };
1576  
1577  /*
1578   * For filesystems using @fs_holder_ops, the @holder argument passed to
1579   * helpers used to open and claim block devices via
1580   * bd_prepare_to_claim() must point to a superblock.
1581   */
1582  extern const struct blk_holder_ops fs_holder_ops;
1583  
1584  /*
1585   * Return the correct open flags for blkdev_get_by_* for super block flags
1586   * as stored in sb->s_flags.
1587   */
1588  #define sb_open_mode(flags) \
1589  	(BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \
1590  	 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
1591  
1592  struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
1593  		const struct blk_holder_ops *hops);
1594  struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,
1595  		void *holder, const struct blk_holder_ops *hops);
1596  int bd_prepare_to_claim(struct block_device *bdev, void *holder,
1597  		const struct blk_holder_ops *hops);
1598  void bd_abort_claiming(struct block_device *bdev, void *holder);
1599  
1600  /* just for blk-cgroup, don't use elsewhere */
1601  struct block_device *blkdev_get_no_open(dev_t dev);
1602  void blkdev_put_no_open(struct block_device *bdev);
1603  
1604  struct block_device *I_BDEV(struct inode *inode);
1605  struct block_device *file_bdev(struct file *bdev_file);
1606  bool disk_live(struct gendisk *disk);
1607  unsigned int block_size(struct block_device *bdev);
1608  
1609  #ifdef CONFIG_BLOCK
1610  void invalidate_bdev(struct block_device *bdev);
1611  int sync_blockdev(struct block_device *bdev);
1612  int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
1613  int sync_blockdev_nowait(struct block_device *bdev);
1614  void sync_bdevs(bool wait);
1615  void bdev_statx(struct path *, struct kstat *, u32);
1616  void printk_all_partitions(void);
1617  int __init early_lookup_bdev(const char *pathname, dev_t *dev);
1618  #else
invalidate_bdev(struct block_device * bdev)1619  static inline void invalidate_bdev(struct block_device *bdev)
1620  {
1621  }
sync_blockdev(struct block_device * bdev)1622  static inline int sync_blockdev(struct block_device *bdev)
1623  {
1624  	return 0;
1625  }
sync_blockdev_nowait(struct block_device * bdev)1626  static inline int sync_blockdev_nowait(struct block_device *bdev)
1627  {
1628  	return 0;
1629  }
sync_bdevs(bool wait)1630  static inline void sync_bdevs(bool wait)
1631  {
1632  }
bdev_statx(struct path * path,struct kstat * stat,u32 request_mask)1633  static inline void bdev_statx(struct path *path, struct kstat *stat,
1634  				u32 request_mask)
1635  {
1636  }
printk_all_partitions(void)1637  static inline void printk_all_partitions(void)
1638  {
1639  }
early_lookup_bdev(const char * pathname,dev_t * dev)1640  static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
1641  {
1642  	return -EINVAL;
1643  }
1644  #endif /* CONFIG_BLOCK */
1645  
1646  int bdev_freeze(struct block_device *bdev);
1647  int bdev_thaw(struct block_device *bdev);
1648  void bdev_fput(struct file *bdev_file);
1649  
1650  struct io_comp_batch {
1651  	struct request *req_list;
1652  	bool need_ts;
1653  	void (*complete)(struct io_comp_batch *);
1654  };
1655  
bdev_can_atomic_write(struct block_device * bdev)1656  static inline bool bdev_can_atomic_write(struct block_device *bdev)
1657  {
1658  	struct request_queue *bd_queue = bdev->bd_queue;
1659  	struct queue_limits *limits = &bd_queue->limits;
1660  
1661  	if (!limits->atomic_write_unit_min)
1662  		return false;
1663  
1664  	if (bdev_is_partition(bdev)) {
1665  		sector_t bd_start_sect = bdev->bd_start_sect;
1666  		unsigned int alignment =
1667  			max(limits->atomic_write_unit_min,
1668  			    limits->atomic_write_hw_boundary);
1669  
1670  		if (!IS_ALIGNED(bd_start_sect, alignment >> SECTOR_SHIFT))
1671  			return false;
1672  	}
1673  
1674  	return true;
1675  }
1676  
1677  #define DEFINE_IO_COMP_BATCH(name)	struct io_comp_batch name = { }
1678  
1679  #endif /* _LINUX_BLKDEV_H */
1680