1 /*
2  * Copyright (c) 2009-2015 Travis Geiselbrecht
3  *
4  * Use of this source code is governed by a MIT-style
5  * license that can be found in the LICENSE file or at
6  * https://opensource.org/licenses/MIT
7  */
8 #include <lib/bio.h>
9 
10 #include <stdlib.h>
11 #include <lk/debug.h>
12 #include <lk/trace.h>
13 #include <lk/err.h>
14 #include <string.h>
15 #include <assert.h>
16 #include <lk/list.h>
17 #include <lk/pow2.h>
18 #include <kernel/mutex.h>
19 #include <lk/init.h>
20 #include <arch/atomic.h>
21 
22 #define LOCAL_TRACE 0
23 
24 static struct {
25     struct list_node list;
26     mutex_t lock;
27 } bdevs = {
28     .list = LIST_INITIAL_VALUE(bdevs.list),
29     .lock = MUTEX_INITIAL_VALUE(bdevs.lock),
30 };
31 
32 /* default implementation is to use the read_block hook to 'deblock' the device */
bio_default_read(struct bdev * dev,void * _buf,off_t offset,size_t len)33 static ssize_t bio_default_read(struct bdev *dev, void *_buf, off_t offset, size_t len) {
34     uint8_t *buf = (uint8_t *)_buf;
35     ssize_t bytes_read = 0;
36     bnum_t block;
37     ssize_t err = 0;
38     STACKBUF_DMA_ALIGN(temp, dev->block_size); // temporary buffer for partial block transfers
39 
40     /* find the starting block */
41     block = offset / dev->block_size;
42 
43     LTRACEF("buf %p, offset %lld, block %u, len %zd\n", buf, offset, block, len);
44     /* handle partial first block */
45     if ((offset % dev->block_size) != 0) {
46         /* read in the block */
47         err = bio_read_block(dev, temp, block, 1);
48         if (err < 0) {
49             goto err;
50         } else if ((size_t)err != dev->block_size) {
51             err = ERR_IO;
52             goto err;
53         }
54 
55         /* copy what we need */
56         size_t block_offset = offset % dev->block_size;
57         size_t tocopy = MIN(dev->block_size - block_offset, len);
58         memcpy(buf, temp + block_offset, tocopy);
59 
60         /* increment our buffers */
61         buf += tocopy;
62         len -= tocopy;
63         bytes_read += tocopy;
64         block++;
65     }
66 
67     LTRACEF("buf %p, block %u, len %zd\n", buf, block, len);
68 
69     // If the device requires alignment AND our buffer is not alread aligned.
70     bool requires_alignment =
71         (dev->flags & BIO_FLAG_CACHE_ALIGNED_READS) &&
72         (IS_ALIGNED((size_t)buf, CACHE_LINE) == false);
73     /* handle middle blocks */
74     if (requires_alignment) {
75         while (len >= dev->block_size) {
76             /* do the middle reads */
77             err = bio_read_block(dev, temp, block, 1);
78             if (err < 0) {
79                 goto err;
80             } else if ((size_t)err != dev->block_size) {
81                 err = ERR_IO;
82                 goto err;
83             }
84             memcpy(buf, temp, dev->block_size);
85 
86             buf += dev->block_size;
87             len -= dev->block_size;
88             bytes_read += dev->block_size;
89             block++;
90         }
91     } else {
92         uint32_t num_blocks = divpow2(len, dev->block_shift);
93         err = bio_read_block(dev, buf, block, num_blocks);
94         if (err < 0) {
95             goto err;
96         } else if ((size_t)err != dev->block_size * num_blocks) {
97             err = ERR_IO;
98             goto err;
99         }
100         buf += err;
101         len -= err;
102         bytes_read += err;
103         block += num_blocks;
104     }
105 
106     LTRACEF("buf %p, block %u, len %zd\n", buf, block, len);
107     /* handle partial last block */
108     if (len > 0) {
109         /* read the block */
110         err = bio_read_block(dev, temp, block, 1);
111         if (err < 0) {
112             goto err;
113         } else if ((size_t)err != dev->block_size) {
114             err = ERR_IO;
115             goto err;
116         }
117 
118         /* copy the partial block from our temp buffer */
119         memcpy(buf, temp, len);
120 
121         bytes_read += len;
122     }
123 
124 err:
125     /* return error or bytes read */
126     return (err >= 0) ? bytes_read : err;
127 }
128 
bio_default_write(struct bdev * dev,const void * _buf,off_t offset,size_t len)129 static ssize_t bio_default_write(struct bdev *dev, const void *_buf, off_t offset, size_t len) {
130     const uint8_t *buf = (const uint8_t *)_buf;
131     ssize_t bytes_written = 0;
132     bnum_t block;
133     ssize_t err = 0;
134     STACKBUF_DMA_ALIGN(temp, dev->block_size); // temporary buffer for partial block transfers
135 
136     /* find the starting block */
137     block = offset / dev->block_size;
138 
139     LTRACEF("buf %p, offset %lld, block %u, len %zd\n", buf, offset, block, len);
140     /* handle partial first block */
141     if ((offset % dev->block_size) != 0) {
142         /* read in the block */
143         err = bio_read_block(dev, temp, block, 1);
144         if (err < 0) {
145             goto err;
146         } else if ((size_t)err != dev->block_size) {
147             err = ERR_IO;
148             goto err;
149         }
150 
151         /* copy what we need */
152         size_t block_offset = offset % dev->block_size;
153         size_t tocopy = MIN(dev->block_size - block_offset, len);
154         memcpy(temp + block_offset, buf, tocopy);
155 
156         /* write it back out */
157         err = bio_write_block(dev, temp, block, 1);
158         if (err < 0) {
159             goto err;
160         } else if ((size_t)err != dev->block_size) {
161             err = ERR_IO;
162             goto err;
163         }
164 
165         /* increment our buffers */
166         buf += tocopy;
167         len -= tocopy;
168         bytes_written += tocopy;
169         block++;
170     }
171 
172     LTRACEF("buf %p, block %u, len %zd\n", buf, block, len);
173 
174     // If the device requires alignment AND our buffer is not alread aligned.
175     bool requires_alignment =
176         (dev->flags & BIO_FLAG_CACHE_ALIGNED_WRITES) &&
177         (IS_ALIGNED((size_t)buf, CACHE_LINE) == false);
178 
179     /* handle middle blocks */
180     if (requires_alignment) {
181         while (len >= dev->block_size) {
182             /* do the middle reads */
183             memcpy(temp, buf, dev->block_size);
184             err = bio_write_block(dev, temp, block, 1);
185             if (err < 0) {
186                 goto err;
187             } else if ((size_t)err != dev->block_size) {
188                 err = ERR_IO;
189                 goto err;
190             }
191 
192             buf += dev->block_size;
193             len -= dev->block_size;
194             bytes_written += dev->block_size;
195             block++;
196         }
197     } else {
198         uint32_t block_count = divpow2(len, dev->block_shift);
199         err = bio_write_block(dev, buf, block, block_count);
200         if (err < 0) {
201             goto err;
202         } else if ((size_t)err != dev->block_size * block_count) {
203             err = ERR_IO;
204             goto err;
205         }
206 
207         DEBUG_ASSERT((size_t)err == (block_count * dev->block_size));
208 
209         buf += err;
210         len -= err;
211         bytes_written += err;
212         block += block_count;
213     }
214 
215     LTRACEF("buf %p, block %u, len %zd\n", buf, block, len);
216     /* handle partial last block */
217     if (len > 0) {
218         /* read the block */
219         err = bio_read_block(dev, temp, block, 1);
220         if (err < 0) {
221             goto err;
222         } else if ((size_t)err != dev->block_size) {
223             err = ERR_IO;
224             goto err;
225         }
226 
227         /* copy the partial block from our temp buffer */
228         memcpy(temp, buf, len);
229 
230         /* write it back out */
231         err = bio_write_block(dev, temp, block, 1);
232         if (err < 0) {
233             goto err;
234         } else if ((size_t)err != dev->block_size) {
235             err = ERR_IO;
236             goto err;
237         }
238 
239         bytes_written += len;
240     }
241 
242 err:
243     /* return error or bytes written */
244     return (err >= 0) ? bytes_written : err;
245 }
246 
bio_default_erase(struct bdev * dev,off_t offset,size_t len)247 static ssize_t bio_default_erase(struct bdev *dev, off_t offset, size_t len) {
248     /* default erase operation is to just write zeros over the device */
249     STACKBUF_DMA_ALIGN(erase_buf, dev->block_size);
250 
251     memset(erase_buf, dev->erase_byte, dev->block_size);
252 
253     ssize_t erased = 0;
254     size_t remaining = len;
255     off_t pos = offset;
256     while (remaining > 0) {
257         size_t towrite = MIN(remaining, dev->block_size);
258 
259         ssize_t written = bio_write(dev, erase_buf, pos, towrite);
260         if (written < 0)
261             return written;
262 
263         erased += written;
264         pos += written;
265         remaining -= written;
266 
267         if ((size_t)written < towrite)
268             break;
269     }
270 
271     return erased;
272 }
273 
bio_default_read_block(struct bdev * dev,void * buf,bnum_t block,uint count)274 static ssize_t bio_default_read_block(struct bdev *dev, void *buf, bnum_t block, uint count) {
275     return ERR_NOT_SUPPORTED;
276 }
277 
bio_default_write_block(struct bdev * dev,const void * buf,bnum_t block,uint count)278 static ssize_t bio_default_write_block(struct bdev *dev, const void *buf, bnum_t block, uint count) {
279     return ERR_NOT_SUPPORTED;
280 }
281 
bdev_inc_ref(bdev_t * dev)282 static void bdev_inc_ref(bdev_t *dev) {
283     LTRACEF("Add ref \"%s\" %d -> %d\n", dev->name, dev->ref, dev->ref + 1);
284     atomic_add(&dev->ref, 1);
285 }
286 
bdev_dec_ref(bdev_t * dev)287 static void bdev_dec_ref(bdev_t *dev) {
288     int oldval = atomic_add(&dev->ref, -1);
289 
290     LTRACEF("Dec ref \"%s\" %d -> %d\n", dev->name, oldval, dev->ref);
291 
292     if (oldval == 1) {
293         // last ref, remove it
294         DEBUG_ASSERT(!list_in_list(&dev->node));
295 
296         TRACEF("last ref, removing (%s)\n", dev->name);
297 
298         // call the close hook if it exists
299         if (dev->close)
300             dev->close(dev);
301 
302         free(dev->name);
303     }
304 }
305 
bio_trim_range(const bdev_t * dev,off_t offset,size_t len)306 size_t bio_trim_range(const bdev_t *dev, off_t offset, size_t len) {
307     /* range check */
308     if (offset < 0)
309         return 0;
310     if (offset >= dev->total_size)
311         return 0;
312     if (len == 0)
313         return 0;
314     if ((off_t)(offset + len) > dev->total_size)
315         len = dev->total_size - offset;
316 
317     return len;
318 }
319 
bio_trim_block_range(const bdev_t * dev,bnum_t block,uint count)320 uint bio_trim_block_range(const bdev_t *dev, bnum_t block, uint count) {
321     if (block > dev->block_count)
322         return 0;
323     if (count == 0)
324         return 0;
325     if (block + count > dev->block_count)
326         count = dev->block_count - block;
327 
328     return count;
329 }
330 
bio_open(const char * name)331 bdev_t *bio_open(const char *name) {
332     bdev_t *bdev = NULL;
333 
334     LTRACEF(" '%s'\n", name);
335 
336     /* see if it's in our list */
337     bdev_t *entry;
338     mutex_acquire(&bdevs.lock);
339     list_for_every_entry(&bdevs.list, entry, bdev_t, node) {
340         DEBUG_ASSERT(entry->ref > 0);
341         if (!strcmp(entry->name, name)) {
342             bdev = entry;
343             bdev_inc_ref(bdev);
344             break;
345         }
346     }
347     mutex_release(&bdevs.lock);
348 
349     return bdev;
350 }
351 
bio_close(bdev_t * dev)352 void bio_close(bdev_t *dev) {
353     DEBUG_ASSERT(dev);
354     LTRACEF(" '%s'\n", dev->name);
355     bdev_dec_ref(dev);
356 }
357 
bio_read(bdev_t * dev,void * buf,off_t offset,size_t len)358 ssize_t bio_read(bdev_t *dev, void *buf, off_t offset, size_t len) {
359     LTRACEF("dev '%s', buf %p, offset %lld, len %zd\n", dev->name, buf, offset, len);
360 
361     DEBUG_ASSERT(dev && dev->ref > 0);
362     DEBUG_ASSERT(buf);
363 
364     /* range check */
365     len = bio_trim_range(dev, offset, len);
366     if (len == 0)
367         return 0;
368 
369     return dev->read(dev, buf, offset, len);
370 }
371 
bio_read_block(bdev_t * dev,void * buf,bnum_t block,uint count)372 ssize_t bio_read_block(bdev_t *dev, void *buf, bnum_t block, uint count) {
373     LTRACEF("dev '%s', buf %p, block %d, count %u\n", dev->name, buf, block, count);
374 
375     DEBUG_ASSERT(dev && dev->ref > 0);
376     DEBUG_ASSERT(buf);
377 
378     /* range check */
379     count = bio_trim_block_range(dev, block, count);
380     if (count == 0)
381         return 0;
382 
383     return dev->read_block(dev, buf, block, count);
384 }
385 
bio_write(bdev_t * dev,const void * buf,off_t offset,size_t len)386 ssize_t bio_write(bdev_t *dev, const void *buf, off_t offset, size_t len) {
387     LTRACEF("dev '%s', buf %p, offset %lld, len %zd\n", dev->name, buf, offset, len);
388 
389     DEBUG_ASSERT(dev && dev->ref > 0);
390     DEBUG_ASSERT(buf);
391 
392     /* range check */
393     len = bio_trim_range(dev, offset, len);
394     if (len == 0)
395         return 0;
396 
397     return dev->write(dev, buf, offset, len);
398 }
399 
bio_write_block(bdev_t * dev,const void * buf,bnum_t block,uint count)400 ssize_t bio_write_block(bdev_t *dev, const void *buf, bnum_t block, uint count) {
401     LTRACEF("dev '%s', buf %p, block %d, count %u\n", dev->name, buf, block, count);
402 
403     DEBUG_ASSERT(dev && dev->ref > 0);
404     DEBUG_ASSERT(buf);
405 
406     /* range check */
407     count = bio_trim_block_range(dev, block, count);
408     if (count == 0)
409         return 0;
410 
411     return dev->write_block(dev, buf, block, count);
412 }
413 
bio_erase(bdev_t * dev,off_t offset,size_t len)414 ssize_t bio_erase(bdev_t *dev, off_t offset, size_t len) {
415     LTRACEF("dev '%s', offset %lld, len %zd\n", dev->name, offset, len);
416 
417     DEBUG_ASSERT(dev && dev->ref > 0);
418 
419     /* range check */
420     len = bio_trim_range(dev, offset, len);
421     if (len == 0)
422         return 0;
423 
424     return dev->erase(dev, offset, len);
425 }
426 
bio_ioctl(bdev_t * dev,int request,void * argp)427 int bio_ioctl(bdev_t *dev, int request, void *argp) {
428     LTRACEF("dev '%s', request %08x, argp %p\n", dev->name, request, argp);
429 
430     if (dev->ioctl == NULL) {
431         return ERR_NOT_SUPPORTED;
432     } else {
433         return dev->ioctl(dev, request, argp);
434     }
435 }
436 
bio_initialize_bdev(bdev_t * dev,const char * name,size_t block_size,bnum_t block_count,size_t geometry_count,const bio_erase_geometry_info_t * geometry,const uint32_t flags)437 void bio_initialize_bdev(bdev_t *dev,
438                          const char *name,
439                          size_t block_size,
440                          bnum_t block_count,
441                          size_t geometry_count,
442                          const bio_erase_geometry_info_t *geometry,
443                          const uint32_t flags) {
444     DEBUG_ASSERT(dev);
445     DEBUG_ASSERT(name);
446 
447     // Block size must be finite powers of 2
448     DEBUG_ASSERT(block_size && ispow2(block_size));
449 
450     list_clear_node(&dev->node);
451     dev->name = strdup(name);
452     dev->block_size = block_size;
453     dev->block_count = block_count;
454     dev->block_shift = log2_uint(block_size);
455     dev->total_size = (off_t)block_count << dev->block_shift;
456     dev->geometry_count = geometry_count;
457     dev->geometry = geometry;
458     dev->erase_byte = 0;
459     dev->ref = 0;
460     dev->flags = flags;
461 
462 #if DEBUG
463     // If we have been supplied information about our erase geometry, sanity
464     // check it in debug builds.
465     if (geometry_count && geometry) {
466         for (size_t i = 0; i < geometry_count; ++i) {
467             bio_erase_geometry_info_t *info = geometry + i;
468 
469             // Erase sizes must be powers of two and agree with the supplied erase shift.
470             DEBUG_ASSERT(info->erase_size);
471             DEBUG_ASSERT(info->erase_size == ((size_t)1 << info->erase_shift));
472 
473             info->start       = desc->start;
474             info->erase_size  = desc->erase_size;
475             info->erase_shift = log2_uint(desc->erase_size);
476             info->size        = ((off_t)desc->block_count) << desc->block_size;
477 
478             // Make sure that region is aligned on both a program and erase block boundary.
479             DEBUG_ASSERT(!(info->start & (((off_t)1 << info->block_shift) - 1)));
480             DEBUG_ASSERT(!(info->start & (((off_t)1 << info->erase_shift) - 1)));
481 
482             // Make sure that region's length is an integral multiple of both the
483             // program and erase block size.
484             DEBUG_ASSERT(!(info->size & (((off_t)1 << dev->block_shift) - 1)));
485             DEBUG_ASSERT(!(info->size & (((off_t)1 << info->erase_shift) - 1)));
486         }
487 
488         // Make sure that none of the regions overlap each other and that they are
489         // listed in ascending order.
490         for (size_t i = 0; (i + 1) < geometry_count; ++i) {
491             bio_geometry_info_t *r1 = dev->geometry + i;
492             bio_geometry_info_t *r2 = dev->geometry + i + 1;
493             DEBUG_ASSERT(r1->start <= r2->start);
494 
495             for (size_t j = (i + 1); j < geometry_count; ++j) {
496                 bio_geometry_info_t *r2 = dev->geometry + j;
497                 DEBUG_ASSERT(!bio_does_overlap(r1->start, r1->size, r2->start, r2->size));
498             }
499         }
500     }
501 #endif
502 
503     /* set up the default hooks, the sub driver should override the block operations at least */
504     dev->read = bio_default_read;
505     dev->read_block = bio_default_read_block;
506     dev->write = bio_default_write;
507     dev->write_block = bio_default_write_block;
508     dev->erase = bio_default_erase;
509     dev->close = NULL;
510 }
511 
bio_register_device(bdev_t * dev)512 void bio_register_device(bdev_t *dev) {
513     DEBUG_ASSERT(dev);
514 
515     LTRACEF(" '%s'\n", dev->name);
516 
517     bdev_inc_ref(dev);
518 
519     mutex_acquire(&bdevs.lock);
520     list_add_tail(&bdevs.list, &dev->node);
521     mutex_release(&bdevs.lock);
522 }
523 
bio_unregister_device(bdev_t * dev)524 void bio_unregister_device(bdev_t *dev) {
525     DEBUG_ASSERT(dev);
526 
527     LTRACEF(" '%s'\n", dev->name);
528 
529     // remove it from the list
530     mutex_acquire(&bdevs.lock);
531     list_delete(&dev->node);
532     mutex_release(&bdevs.lock);
533 
534     bdev_dec_ref(dev); // remove the ref the list used to have
535 }
536 
bio_dump_devices(void)537 void bio_dump_devices(void) {
538     printf("block devices:\n");
539     bdev_t *entry;
540     mutex_acquire(&bdevs.lock);
541     list_for_every_entry(&bdevs.list, entry, bdev_t, node) {
542 
543         printf("\t%s, size %lld, bsize %zd, ref %d",
544                entry->name, entry->total_size, entry->block_size, entry->ref);
545 
546         if (!entry->geometry_count || !entry->geometry) {
547             printf(" (no erase geometry)\n");
548         } else {
549             for (size_t i = 0; i < entry->geometry_count; ++i) {
550                 const bio_erase_geometry_info_t *geo = entry->geometry + i;
551                 printf("\n\t\terase_region[%zu] : start %lld size %lld erase size %zu",
552                        i, geo->start, geo->size, geo->erase_size);
553 
554             }
555         }
556 
557         printf("\n");
558     }
559     mutex_release(&bdevs.lock);
560 }
561