1 // SPDX-License-Identifier: GPL-2.0-or-later
2
3 #define pr_fmt(fmt) "ref_tracker: " fmt
4
5 #include <linux/export.h>
6 #include <linux/list_sort.h>
7 #include <linux/ref_tracker.h>
8 #include <linux/slab.h>
9 #include <linux/stacktrace.h>
10 #include <linux/stackdepot.h>
11 #include <linux/seq_file.h>
12
13 #define REF_TRACKER_STACK_ENTRIES 16
14 #define STACK_BUF_SIZE 1024
15
16 struct ref_tracker {
17 struct list_head head; /* anchor into dir->list or dir->quarantine */
18 bool dead;
19 depot_stack_handle_t alloc_stack_handle;
20 depot_stack_handle_t free_stack_handle;
21 };
22
23 struct ref_tracker_dir_stats {
24 int total;
25 int count;
26 struct {
27 depot_stack_handle_t stack_handle;
28 unsigned int count;
29 } stacks[];
30 };
31
32 #ifdef CONFIG_DEBUG_FS
33 #include <linux/xarray.h>
34
35 /*
36 * ref_tracker_dir_init() is usually called in allocation-safe contexts, but
37 * the same is not true of ref_tracker_dir_exit() which can be called from
38 * anywhere an object is freed. Removing debugfs dentries is a blocking
39 * operation, so we defer that work to the debugfs_reap_worker.
40 *
41 * Each dentry is tracked in the appropriate xarray. When
42 * ref_tracker_dir_exit() is called, its entries in the xarrays are marked and
43 * the workqueue job is scheduled. The worker then runs and deletes any marked
44 * dentries asynchronously.
45 */
46 static struct xarray debugfs_dentries;
47 static struct xarray debugfs_symlinks;
48 static struct work_struct debugfs_reap_worker;
49
50 #define REF_TRACKER_DIR_DEAD XA_MARK_0
ref_tracker_debugfs_mark(struct ref_tracker_dir * dir)51 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir)
52 {
53 unsigned long flags;
54
55 xa_lock_irqsave(&debugfs_dentries, flags);
56 __xa_set_mark(&debugfs_dentries, (unsigned long)dir, REF_TRACKER_DIR_DEAD);
57 xa_unlock_irqrestore(&debugfs_dentries, flags);
58
59 xa_lock_irqsave(&debugfs_symlinks, flags);
60 __xa_set_mark(&debugfs_symlinks, (unsigned long)dir, REF_TRACKER_DIR_DEAD);
61 xa_unlock_irqrestore(&debugfs_symlinks, flags);
62
63 schedule_work(&debugfs_reap_worker);
64 }
65 #else
ref_tracker_debugfs_mark(struct ref_tracker_dir * dir)66 static inline void ref_tracker_debugfs_mark(struct ref_tracker_dir *dir)
67 {
68 }
69 #endif
70
71 static struct ref_tracker_dir_stats *
ref_tracker_get_stats(struct ref_tracker_dir * dir,unsigned int limit)72 ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit)
73 {
74 struct ref_tracker_dir_stats *stats;
75 struct ref_tracker *tracker;
76
77 stats = kmalloc(struct_size(stats, stacks, limit),
78 GFP_NOWAIT | __GFP_NOWARN);
79 if (!stats)
80 return ERR_PTR(-ENOMEM);
81 stats->total = 0;
82 stats->count = 0;
83
84 list_for_each_entry(tracker, &dir->list, head) {
85 depot_stack_handle_t stack = tracker->alloc_stack_handle;
86 int i;
87
88 ++stats->total;
89 for (i = 0; i < stats->count; ++i)
90 if (stats->stacks[i].stack_handle == stack)
91 break;
92 if (i >= limit)
93 continue;
94 if (i >= stats->count) {
95 stats->stacks[i].stack_handle = stack;
96 stats->stacks[i].count = 0;
97 ++stats->count;
98 }
99 ++stats->stacks[i].count;
100 }
101
102 return stats;
103 }
104
105 struct ostream {
106 void __ostream_printf (*func)(struct ostream *stream, char *fmt, ...);
107 char *prefix;
108 char *buf;
109 struct seq_file *seq;
110 int size, used;
111 };
112
pr_ostream_log(struct ostream * stream,char * fmt,...)113 static void __ostream_printf pr_ostream_log(struct ostream *stream, char *fmt, ...)
114 {
115 va_list args;
116
117 va_start(args, fmt);
118 vprintk(fmt, args);
119 va_end(args);
120 }
121
pr_ostream_buf(struct ostream * stream,char * fmt,...)122 static void __ostream_printf pr_ostream_buf(struct ostream *stream, char *fmt, ...)
123 {
124 int ret, len = stream->size - stream->used;
125 va_list args;
126
127 va_start(args, fmt);
128 ret = vsnprintf(stream->buf + stream->used, len, fmt, args);
129 va_end(args);
130 if (ret > 0)
131 stream->used += min(ret, len);
132 }
133
134 #define pr_ostream(stream, fmt, args...) \
135 ({ \
136 struct ostream *_s = (stream); \
137 \
138 _s->func(_s, fmt, ##args); \
139 })
140
141 static void
__ref_tracker_dir_pr_ostream(struct ref_tracker_dir * dir,unsigned int display_limit,struct ostream * s)142 __ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir,
143 unsigned int display_limit, struct ostream *s)
144 {
145 struct ref_tracker_dir_stats *stats;
146 unsigned int i = 0, skipped;
147 depot_stack_handle_t stack;
148 char *sbuf;
149
150 lockdep_assert_held(&dir->lock);
151
152 if (list_empty(&dir->list))
153 return;
154
155 stats = ref_tracker_get_stats(dir, display_limit);
156 if (IS_ERR(stats)) {
157 pr_ostream(s, "%s%s@%p: couldn't get stats, error %pe\n",
158 s->prefix, dir->class, dir, stats);
159 return;
160 }
161
162 sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT | __GFP_NOWARN);
163
164 for (i = 0, skipped = stats->total; i < stats->count; ++i) {
165 stack = stats->stacks[i].stack_handle;
166 if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4))
167 sbuf[0] = 0;
168 pr_ostream(s, "%s%s@%p has %d/%d users at\n%s\n", s->prefix,
169 dir->class, dir, stats->stacks[i].count,
170 stats->total, sbuf);
171 skipped -= stats->stacks[i].count;
172 }
173
174 if (skipped)
175 pr_ostream(s, "%s%s@%p skipped reports about %d/%d users.\n",
176 s->prefix, dir->class, dir, skipped, stats->total);
177
178 kfree(sbuf);
179
180 kfree(stats);
181 }
182
ref_tracker_dir_print_locked(struct ref_tracker_dir * dir,unsigned int display_limit)183 void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir,
184 unsigned int display_limit)
185 {
186 struct ostream os = { .func = pr_ostream_log,
187 .prefix = "ref_tracker: " };
188
189 __ref_tracker_dir_pr_ostream(dir, display_limit, &os);
190 }
191 EXPORT_SYMBOL(ref_tracker_dir_print_locked);
192
ref_tracker_dir_print(struct ref_tracker_dir * dir,unsigned int display_limit)193 void ref_tracker_dir_print(struct ref_tracker_dir *dir,
194 unsigned int display_limit)
195 {
196 unsigned long flags;
197
198 spin_lock_irqsave(&dir->lock, flags);
199 ref_tracker_dir_print_locked(dir, display_limit);
200 spin_unlock_irqrestore(&dir->lock, flags);
201 }
202 EXPORT_SYMBOL(ref_tracker_dir_print);
203
ref_tracker_dir_snprint(struct ref_tracker_dir * dir,char * buf,size_t size)204 int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size)
205 {
206 struct ostream os = { .func = pr_ostream_buf,
207 .prefix = "ref_tracker: ",
208 .buf = buf,
209 .size = size };
210 unsigned long flags;
211
212 spin_lock_irqsave(&dir->lock, flags);
213 __ref_tracker_dir_pr_ostream(dir, 16, &os);
214 spin_unlock_irqrestore(&dir->lock, flags);
215
216 return os.used;
217 }
218 EXPORT_SYMBOL(ref_tracker_dir_snprint);
219
ref_tracker_dir_exit(struct ref_tracker_dir * dir)220 void ref_tracker_dir_exit(struct ref_tracker_dir *dir)
221 {
222 struct ref_tracker *tracker, *n;
223 unsigned long flags;
224 bool leak = false;
225
226 dir->dead = true;
227 /*
228 * The xarray entries must be marked before the dir->lock is taken to
229 * protect simultaneous debugfs readers.
230 */
231 ref_tracker_debugfs_mark(dir);
232 spin_lock_irqsave(&dir->lock, flags);
233 list_for_each_entry_safe(tracker, n, &dir->quarantine, head) {
234 list_del(&tracker->head);
235 kfree(tracker);
236 dir->quarantine_avail++;
237 }
238 if (!list_empty(&dir->list)) {
239 ref_tracker_dir_print_locked(dir, 16);
240 leak = true;
241 list_for_each_entry_safe(tracker, n, &dir->list, head) {
242 list_del(&tracker->head);
243 kfree(tracker);
244 }
245 }
246 spin_unlock_irqrestore(&dir->lock, flags);
247 WARN_ON_ONCE(leak);
248 WARN_ON_ONCE(refcount_read(&dir->untracked) != 1);
249 WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1);
250 }
251 EXPORT_SYMBOL(ref_tracker_dir_exit);
252
ref_tracker_alloc(struct ref_tracker_dir * dir,struct ref_tracker ** trackerp,gfp_t gfp)253 int ref_tracker_alloc(struct ref_tracker_dir *dir,
254 struct ref_tracker **trackerp,
255 gfp_t gfp)
256 {
257 unsigned long entries[REF_TRACKER_STACK_ENTRIES];
258 struct ref_tracker *tracker;
259 unsigned int nr_entries;
260 gfp_t gfp_mask = gfp | __GFP_NOWARN;
261 unsigned long flags;
262
263 WARN_ON_ONCE(dir->dead);
264
265 if (!trackerp) {
266 refcount_inc(&dir->no_tracker);
267 return 0;
268 }
269 if (gfp & __GFP_DIRECT_RECLAIM)
270 gfp_mask |= __GFP_NOFAIL;
271 *trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask);
272 if (unlikely(!tracker)) {
273 pr_err_once("memory allocation failure, unreliable refcount tracker.\n");
274 refcount_inc(&dir->untracked);
275 return -ENOMEM;
276 }
277 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
278 tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp);
279
280 spin_lock_irqsave(&dir->lock, flags);
281 list_add(&tracker->head, &dir->list);
282 spin_unlock_irqrestore(&dir->lock, flags);
283 return 0;
284 }
285 EXPORT_SYMBOL_GPL(ref_tracker_alloc);
286
ref_tracker_free(struct ref_tracker_dir * dir,struct ref_tracker ** trackerp)287 int ref_tracker_free(struct ref_tracker_dir *dir,
288 struct ref_tracker **trackerp)
289 {
290 unsigned long entries[REF_TRACKER_STACK_ENTRIES];
291 depot_stack_handle_t stack_handle;
292 struct ref_tracker *tracker;
293 unsigned int nr_entries;
294 unsigned long flags;
295
296 WARN_ON_ONCE(dir->dead);
297
298 if (!trackerp) {
299 refcount_dec(&dir->no_tracker);
300 return 0;
301 }
302 tracker = *trackerp;
303 if (!tracker) {
304 refcount_dec(&dir->untracked);
305 return -EEXIST;
306 }
307 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
308 stack_handle = stack_depot_save(entries, nr_entries,
309 GFP_NOWAIT | __GFP_NOWARN);
310
311 spin_lock_irqsave(&dir->lock, flags);
312 if (tracker->dead) {
313 pr_err("reference already released.\n");
314 if (tracker->alloc_stack_handle) {
315 pr_err("allocated in:\n");
316 stack_depot_print(tracker->alloc_stack_handle);
317 }
318 if (tracker->free_stack_handle) {
319 pr_err("freed in:\n");
320 stack_depot_print(tracker->free_stack_handle);
321 }
322 spin_unlock_irqrestore(&dir->lock, flags);
323 WARN_ON_ONCE(1);
324 return -EINVAL;
325 }
326 tracker->dead = true;
327
328 tracker->free_stack_handle = stack_handle;
329
330 list_move_tail(&tracker->head, &dir->quarantine);
331 if (!dir->quarantine_avail) {
332 tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head);
333 list_del(&tracker->head);
334 } else {
335 dir->quarantine_avail--;
336 tracker = NULL;
337 }
338 spin_unlock_irqrestore(&dir->lock, flags);
339
340 kfree(tracker);
341 return 0;
342 }
343 EXPORT_SYMBOL_GPL(ref_tracker_free);
344
345 #ifdef CONFIG_DEBUG_FS
346 #include <linux/debugfs.h>
347
348 static struct dentry *ref_tracker_debug_dir = (struct dentry *)-ENOENT;
349
pr_ostream_seq(struct ostream * stream,char * fmt,...)350 static void __ostream_printf pr_ostream_seq(struct ostream *stream, char *fmt, ...)
351 {
352 va_list args;
353
354 va_start(args, fmt);
355 seq_vprintf(stream->seq, fmt, args);
356 va_end(args);
357 }
358
ref_tracker_dir_seq_print(struct ref_tracker_dir * dir,struct seq_file * seq)359 static int ref_tracker_dir_seq_print(struct ref_tracker_dir *dir, struct seq_file *seq)
360 {
361 struct ostream os = { .func = pr_ostream_seq,
362 .prefix = "",
363 .seq = seq };
364
365 __ref_tracker_dir_pr_ostream(dir, 16, &os);
366
367 return os.used;
368 }
369
ref_tracker_debugfs_show(struct seq_file * f,void * v)370 static int ref_tracker_debugfs_show(struct seq_file *f, void *v)
371 {
372 struct ref_tracker_dir *dir = f->private;
373 unsigned long index = (unsigned long)dir;
374 unsigned long flags;
375 int ret;
376
377 /*
378 * "dir" may not exist at this point if ref_tracker_dir_exit() has
379 * already been called. Take care not to dereference it until its
380 * legitimacy is established.
381 *
382 * The xa_lock is necessary to ensure that "dir" doesn't disappear
383 * before its lock can be taken. If it's in the hash and not marked
384 * dead, then it's safe to take dir->lock which prevents
385 * ref_tracker_dir_exit() from completing. Once the dir->lock is
386 * acquired, the xa_lock can be released. All of this must be IRQ-safe.
387 */
388 xa_lock_irqsave(&debugfs_dentries, flags);
389 if (!xa_load(&debugfs_dentries, index) ||
390 xa_get_mark(&debugfs_dentries, index, REF_TRACKER_DIR_DEAD)) {
391 xa_unlock_irqrestore(&debugfs_dentries, flags);
392 return -ENODATA;
393 }
394
395 spin_lock(&dir->lock);
396 xa_unlock(&debugfs_dentries);
397 ret = ref_tracker_dir_seq_print(dir, f);
398 spin_unlock_irqrestore(&dir->lock, flags);
399 return ret;
400 }
401
ref_tracker_debugfs_open(struct inode * inode,struct file * filp)402 static int ref_tracker_debugfs_open(struct inode *inode, struct file *filp)
403 {
404 struct ref_tracker_dir *dir = inode->i_private;
405
406 return single_open(filp, ref_tracker_debugfs_show, dir);
407 }
408
409 static const struct file_operations ref_tracker_debugfs_fops = {
410 .owner = THIS_MODULE,
411 .open = ref_tracker_debugfs_open,
412 .read = seq_read,
413 .llseek = seq_lseek,
414 .release = single_release,
415 };
416
417 /**
418 * ref_tracker_dir_debugfs - create debugfs file for ref_tracker_dir
419 * @dir: ref_tracker_dir to be associated with debugfs file
420 *
421 * In most cases, a debugfs file will be created automatically for every
422 * ref_tracker_dir. If the object was created before debugfs is brought up
423 * then that may fail. In those cases, it is safe to call this at a later
424 * time to create the file.
425 */
ref_tracker_dir_debugfs(struct ref_tracker_dir * dir)426 void ref_tracker_dir_debugfs(struct ref_tracker_dir *dir)
427 {
428 char name[NAME_MAX + 1];
429 struct dentry *dentry;
430 int ret;
431
432 /* No-op if already created */
433 dentry = xa_load(&debugfs_dentries, (unsigned long)dir);
434 if (dentry && !xa_is_err(dentry))
435 return;
436
437 ret = snprintf(name, sizeof(name), "%s@%p", dir->class, dir);
438 name[sizeof(name) - 1] = '\0';
439
440 if (ret < sizeof(name)) {
441 dentry = debugfs_create_file(name, S_IFREG | 0400,
442 ref_tracker_debug_dir, dir,
443 &ref_tracker_debugfs_fops);
444 if (!IS_ERR(dentry)) {
445 void *old;
446
447 old = xa_store_irq(&debugfs_dentries, (unsigned long)dir,
448 dentry, GFP_KERNEL);
449
450 if (xa_is_err(old))
451 debugfs_remove(dentry);
452 else
453 WARN_ON_ONCE(old);
454 }
455 }
456 }
457 EXPORT_SYMBOL(ref_tracker_dir_debugfs);
458
ref_tracker_dir_symlink(struct ref_tracker_dir * dir,const char * fmt,...)459 void __ostream_printf ref_tracker_dir_symlink(struct ref_tracker_dir *dir, const char *fmt, ...)
460 {
461 char name[NAME_MAX + 1];
462 struct dentry *symlink, *dentry;
463 va_list args;
464 int ret;
465
466 symlink = xa_load(&debugfs_symlinks, (unsigned long)dir);
467 dentry = xa_load(&debugfs_dentries, (unsigned long)dir);
468
469 /* Already created?*/
470 if (symlink && !xa_is_err(symlink))
471 return;
472
473 if (!dentry || xa_is_err(dentry))
474 return;
475
476 va_start(args, fmt);
477 ret = vsnprintf(name, sizeof(name), fmt, args);
478 va_end(args);
479 name[sizeof(name) - 1] = '\0';
480
481 if (ret < sizeof(name)) {
482 symlink = debugfs_create_symlink(name, ref_tracker_debug_dir,
483 dentry->d_name.name);
484 if (!IS_ERR(symlink)) {
485 void *old;
486
487 old = xa_store_irq(&debugfs_symlinks, (unsigned long)dir,
488 symlink, GFP_KERNEL);
489 if (xa_is_err(old))
490 debugfs_remove(symlink);
491 else
492 WARN_ON_ONCE(old);
493 }
494 }
495 }
496 EXPORT_SYMBOL(ref_tracker_dir_symlink);
497
debugfs_reap_work(struct work_struct * work)498 static void debugfs_reap_work(struct work_struct *work)
499 {
500 struct dentry *dentry;
501 unsigned long index;
502 bool reaped;
503
504 do {
505 reaped = false;
506 xa_for_each_marked(&debugfs_symlinks, index, dentry, REF_TRACKER_DIR_DEAD) {
507 xa_erase_irq(&debugfs_symlinks, index);
508 debugfs_remove(dentry);
509 reaped = true;
510 }
511 xa_for_each_marked(&debugfs_dentries, index, dentry, REF_TRACKER_DIR_DEAD) {
512 xa_erase_irq(&debugfs_dentries, index);
513 debugfs_remove(dentry);
514 reaped = true;
515 }
516 } while (reaped);
517 }
518
ref_tracker_debugfs_postcore_init(void)519 static int __init ref_tracker_debugfs_postcore_init(void)
520 {
521 INIT_WORK(&debugfs_reap_worker, debugfs_reap_work);
522 xa_init_flags(&debugfs_dentries, XA_FLAGS_LOCK_IRQ);
523 xa_init_flags(&debugfs_symlinks, XA_FLAGS_LOCK_IRQ);
524 return 0;
525 }
526 postcore_initcall(ref_tracker_debugfs_postcore_init);
527
ref_tracker_debugfs_late_init(void)528 static int __init ref_tracker_debugfs_late_init(void)
529 {
530 ref_tracker_debug_dir = debugfs_create_dir("ref_tracker", NULL);
531 return 0;
532 }
533 late_initcall(ref_tracker_debugfs_late_init);
534 #endif /* CONFIG_DEBUG_FS */
535