1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/module.h>
4 #include <linux/backing-dev.h>
5 #include <linux/bio.h>
6 #include <linux/blkdev.h>
7 #include <linux/mm.h>
8 #include <linux/init.h>
9 #include <linux/slab.h>
10 #include <linux/workqueue.h>
11 #include <linux/smp.h>
12
13 #include <linux/blk-mq.h>
14 #include "blk.h"
15 #include "blk-mq.h"
16 #include "blk-mq-tag.h"
17
blk_mq_sysfs_release(struct kobject * kobj)18 static void blk_mq_sysfs_release(struct kobject *kobj)
19 {
20 struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);
21
22 free_percpu(ctxs->queue_ctx);
23 kfree(ctxs);
24 }
25
blk_mq_ctx_sysfs_release(struct kobject * kobj)26 static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
27 {
28 struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);
29
30 /* ctx->ctxs won't be released until all ctx are freed */
31 kobject_put(&ctx->ctxs->kobj);
32 }
33
blk_mq_hw_sysfs_release(struct kobject * kobj)34 static void blk_mq_hw_sysfs_release(struct kobject *kobj)
35 {
36 struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
37 kobj);
38
39 blk_free_flush_queue(hctx->fq);
40 sbitmap_free(&hctx->ctx_map);
41 free_cpumask_var(hctx->cpumask);
42 kfree(hctx->ctxs);
43 kfree(hctx);
44 }
45
46 struct blk_mq_hw_ctx_sysfs_entry {
47 struct attribute attr;
48 ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
49 };
50
blk_mq_hw_sysfs_show(struct kobject * kobj,struct attribute * attr,char * page)51 static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
52 struct attribute *attr, char *page)
53 {
54 struct blk_mq_hw_ctx_sysfs_entry *entry;
55 struct blk_mq_hw_ctx *hctx;
56 struct request_queue *q;
57 ssize_t res;
58
59 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
60 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
61 q = hctx->queue;
62
63 if (!entry->show)
64 return -EIO;
65
66 mutex_lock(&q->sysfs_lock);
67 res = entry->show(hctx, page);
68 mutex_unlock(&q->sysfs_lock);
69 return res;
70 }
71
blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx * hctx,char * page)72 static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
73 char *page)
74 {
75 return sprintf(page, "%u\n", hctx->tags->nr_tags);
76 }
77
blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx * hctx,char * page)78 static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
79 char *page)
80 {
81 return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
82 }
83
blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx * hctx,char * page)84 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
85 {
86 const size_t size = PAGE_SIZE - 1;
87 unsigned int i, first = 1;
88 int ret = 0, pos = 0;
89
90 for_each_cpu(i, hctx->cpumask) {
91 if (first)
92 ret = snprintf(pos + page, size - pos, "%u", i);
93 else
94 ret = snprintf(pos + page, size - pos, ", %u", i);
95
96 if (ret >= size - pos)
97 break;
98
99 first = 0;
100 pos += ret;
101 }
102
103 ret = snprintf(pos + page, size + 1 - pos, "\n");
104 return pos + ret;
105 }
106
107 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
108 .attr = {.name = "nr_tags", .mode = 0444 },
109 .show = blk_mq_hw_sysfs_nr_tags_show,
110 };
111 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
112 .attr = {.name = "nr_reserved_tags", .mode = 0444 },
113 .show = blk_mq_hw_sysfs_nr_reserved_tags_show,
114 };
115 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
116 .attr = {.name = "cpu_list", .mode = 0444 },
117 .show = blk_mq_hw_sysfs_cpus_show,
118 };
119
120 static struct attribute *default_hw_ctx_attrs[] = {
121 &blk_mq_hw_sysfs_nr_tags.attr,
122 &blk_mq_hw_sysfs_nr_reserved_tags.attr,
123 &blk_mq_hw_sysfs_cpus.attr,
124 NULL,
125 };
126 ATTRIBUTE_GROUPS(default_hw_ctx);
127
128 static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
129 .show = blk_mq_hw_sysfs_show,
130 };
131
132 static const struct kobj_type blk_mq_ktype = {
133 .release = blk_mq_sysfs_release,
134 };
135
136 static const struct kobj_type blk_mq_ctx_ktype = {
137 .release = blk_mq_ctx_sysfs_release,
138 };
139
140 static const struct kobj_type blk_mq_hw_ktype = {
141 .sysfs_ops = &blk_mq_hw_sysfs_ops,
142 .default_groups = default_hw_ctx_groups,
143 .release = blk_mq_hw_sysfs_release,
144 };
145
blk_mq_unregister_hctx(struct blk_mq_hw_ctx * hctx)146 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
147 {
148 struct blk_mq_ctx *ctx;
149 int i;
150
151 if (!hctx->nr_ctx)
152 return;
153
154 hctx_for_each_ctx(hctx, ctx, i)
155 kobject_del(&ctx->kobj);
156
157 kobject_del(&hctx->kobj);
158 }
159
blk_mq_register_hctx(struct blk_mq_hw_ctx * hctx)160 static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
161 {
162 struct request_queue *q = hctx->queue;
163 struct blk_mq_ctx *ctx;
164 int i, j, ret;
165
166 if (!hctx->nr_ctx)
167 return 0;
168
169 ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
170 if (ret)
171 return ret;
172
173 hctx_for_each_ctx(hctx, ctx, i) {
174 ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
175 if (ret)
176 goto out;
177 }
178
179 return 0;
180 out:
181 hctx_for_each_ctx(hctx, ctx, j) {
182 if (j < i)
183 kobject_del(&ctx->kobj);
184 }
185 kobject_del(&hctx->kobj);
186 return ret;
187 }
188
blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx * hctx)189 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
190 {
191 kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
192 }
193
blk_mq_sysfs_deinit(struct request_queue * q)194 void blk_mq_sysfs_deinit(struct request_queue *q)
195 {
196 struct blk_mq_ctx *ctx;
197 int cpu;
198
199 for_each_possible_cpu(cpu) {
200 ctx = per_cpu_ptr(q->queue_ctx, cpu);
201 kobject_put(&ctx->kobj);
202 }
203 kobject_put(q->mq_kobj);
204 }
205
blk_mq_sysfs_init(struct request_queue * q)206 void blk_mq_sysfs_init(struct request_queue *q)
207 {
208 struct blk_mq_ctx *ctx;
209 int cpu;
210
211 kobject_init(q->mq_kobj, &blk_mq_ktype);
212
213 for_each_possible_cpu(cpu) {
214 ctx = per_cpu_ptr(q->queue_ctx, cpu);
215
216 kobject_get(q->mq_kobj);
217 kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
218 }
219 }
220
blk_mq_sysfs_register(struct gendisk * disk)221 int blk_mq_sysfs_register(struct gendisk *disk)
222 {
223 struct request_queue *q = disk->queue;
224 struct blk_mq_hw_ctx *hctx;
225 unsigned long i, j;
226 int ret;
227
228 lockdep_assert_held(&q->sysfs_dir_lock);
229
230 ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq");
231 if (ret < 0)
232 goto out;
233
234 kobject_uevent(q->mq_kobj, KOBJ_ADD);
235
236 queue_for_each_hw_ctx(q, hctx, i) {
237 ret = blk_mq_register_hctx(hctx);
238 if (ret)
239 goto unreg;
240 }
241
242 q->mq_sysfs_init_done = true;
243
244 out:
245 return ret;
246
247 unreg:
248 queue_for_each_hw_ctx(q, hctx, j) {
249 if (j < i)
250 blk_mq_unregister_hctx(hctx);
251 }
252
253 kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
254 kobject_del(q->mq_kobj);
255 return ret;
256 }
257
blk_mq_sysfs_unregister(struct gendisk * disk)258 void blk_mq_sysfs_unregister(struct gendisk *disk)
259 {
260 struct request_queue *q = disk->queue;
261 struct blk_mq_hw_ctx *hctx;
262 unsigned long i;
263
264 lockdep_assert_held(&q->sysfs_dir_lock);
265
266 queue_for_each_hw_ctx(q, hctx, i)
267 blk_mq_unregister_hctx(hctx);
268
269 kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
270 kobject_del(q->mq_kobj);
271
272 q->mq_sysfs_init_done = false;
273 }
274
blk_mq_sysfs_unregister_hctxs(struct request_queue * q)275 void blk_mq_sysfs_unregister_hctxs(struct request_queue *q)
276 {
277 struct blk_mq_hw_ctx *hctx;
278 unsigned long i;
279
280 mutex_lock(&q->sysfs_dir_lock);
281 if (!q->mq_sysfs_init_done)
282 goto unlock;
283
284 queue_for_each_hw_ctx(q, hctx, i)
285 blk_mq_unregister_hctx(hctx);
286
287 unlock:
288 mutex_unlock(&q->sysfs_dir_lock);
289 }
290
blk_mq_sysfs_register_hctxs(struct request_queue * q)291 int blk_mq_sysfs_register_hctxs(struct request_queue *q)
292 {
293 struct blk_mq_hw_ctx *hctx;
294 unsigned long i;
295 int ret = 0;
296
297 mutex_lock(&q->sysfs_dir_lock);
298 if (!q->mq_sysfs_init_done)
299 goto unlock;
300
301 queue_for_each_hw_ctx(q, hctx, i) {
302 ret = blk_mq_register_hctx(hctx);
303 if (ret)
304 break;
305 }
306
307 unlock:
308 mutex_unlock(&q->sysfs_dir_lock);
309
310 return ret;
311 }
312