1 // Copyright 2017 The Fuchsia Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include <ddk/phys-iter.h>
6 #include <sys/param.h>
7 #include <zircon/assert.h>
8 
9 #include <string.h>
10 
11 // Returns the buffer offset of the start of the current segment being iterated over.
segment_offset(phys_iter_t * iter)12 static size_t segment_offset(phys_iter_t* iter) {
13     // The physical pages list begins with the page containing buf->vmo_offset,
14     // while the stored segment_offset is relative to the buffer vmo offset.
15     return (iter->buf.vmo_offset & (PAGE_SIZE - 1)) + iter->segment_offset;
16 }
17 
18 // Initializes the iterator for the next segment to iterate over.
19 // Returns whether there was a next segment.
init_next_sg_entry(phys_iter_t * iter)20 static bool init_next_sg_entry(phys_iter_t* iter) {
21     // Finished iterating through the scatter gather list.
22     if (iter->buf.sg_list && iter->next_sg_entry_idx >= iter->buf.sg_count) {
23         return false;
24     }
25     // No scatter gather list was provided and we have finished iterating
26     // over the requested length.
27     if (!iter->buf.sg_list && iter->total_iterated == iter->buf.length) {
28         return false;
29     }
30     if (iter->buf.sg_list) {
31         phys_iter_sg_entry_t* next = &iter->buf.sg_list[iter->next_sg_entry_idx];
32         iter->segment_length = next->length;
33         iter->segment_offset = next->offset;
34         iter->next_sg_entry_idx++;
35     } else {
36         iter->segment_length = iter->buf.length;
37         iter->segment_offset = 0;
38     }
39 
40     iter->offset = 0;
41     // iter->page is index of page containing the next segment start offset.
42     // and iter->last_page is index of page containing the segment offset + segment->length
43     iter->page = iter->buf.phys_count == 1 ? 0 : segment_offset(iter) / PAGE_SIZE;
44     if (iter->segment_length > 0) {
45         iter->last_page = (iter->segment_length + segment_offset(iter) - 1) / PAGE_SIZE;
46         iter->last_page = MIN(iter->last_page, iter->buf.phys_count - 1);
47     } else {
48         iter->last_page = 0;
49     }
50     return true;
51 }
52 
phys_iter_init(phys_iter_t * iter,phys_iter_buffer_t * buf,size_t max_length)53 void phys_iter_init(phys_iter_t* iter, phys_iter_buffer_t* buf, size_t max_length) {
54     memset(iter, 0, sizeof(phys_iter_t));
55     memcpy(&iter->buf, buf, sizeof(phys_iter_buffer_t));
56     ZX_DEBUG_ASSERT(max_length % PAGE_SIZE == 0);
57     if (max_length == 0) {
58         max_length = UINT64_MAX;
59     }
60     iter->max_length = max_length;
61     init_next_sg_entry(iter);
62 }
63 
phys_iter_increment(phys_iter_t * iter,size_t len)64 static inline void phys_iter_increment(phys_iter_t* iter, size_t len) {
65     iter->total_iterated += len;
66     iter->offset += len;
67 }
68 
phys_iter_next(phys_iter_t * iter,zx_paddr_t * out_paddr)69 size_t phys_iter_next(phys_iter_t* iter, zx_paddr_t* out_paddr) {
70     // Check if we've finished iterating over the current segment.
71     // We shouldn't have any zero length segments, but use a while loop just in case.
72     while (iter->offset >= iter->segment_length) {
73         bool has_next = init_next_sg_entry(iter);
74         if (!has_next) {
75             return 0;
76         }
77     }
78 
79     phys_iter_buffer_t buf = iter->buf;
80     zx_off_t offset = iter->offset;
81     size_t max_length = iter->max_length;
82     size_t length = iter->segment_length;
83 
84     size_t remaining = length - offset;
85     zx_paddr_t* phys_addrs = buf.phys;
86     size_t align_adjust = segment_offset(iter) & (PAGE_SIZE - 1);
87     zx_paddr_t phys = phys_addrs[iter->page];
88     size_t return_length = 0;
89 
90     if (buf.phys_count == 1) {
91         // simple contiguous case
92         *out_paddr = phys_addrs[0] + offset + segment_offset(iter);
93         return_length = remaining;
94         if (return_length > max_length) {
95             // end on a page boundary
96             return_length = max_length - align_adjust;
97         }
98         phys_iter_increment(iter, return_length);
99         return return_length;
100     }
101 
102     if (offset == 0 && align_adjust > 0) {
103         // if the segment offset is unaligned we need to adjust out_paddr, accumulate partial
104         // page length in return_length and skip to next page.
105         // we will make sure the range ends on a page boundary so we don't need to worry about
106         // alignment for subsequent iterations.
107         *out_paddr = phys + align_adjust;
108         return_length = MIN(PAGE_SIZE - align_adjust, remaining);
109         remaining -= return_length;
110         iter->page++;
111 
112         if (iter->page > iter->last_page || phys + PAGE_SIZE != phys_addrs[iter->page]) {
113             phys_iter_increment(iter, return_length);
114             return return_length;
115         }
116         phys = phys_addrs[iter->page];
117     } else {
118         *out_paddr = phys;
119     }
120 
121     // below is more complicated case where we need to watch for discontinuities
122     // in the physical address space.
123 
124     // loop through physical addresses looking for discontinuities
125     while (remaining > 0 && iter->page <= iter->last_page) {
126         const size_t increment = MIN(PAGE_SIZE, remaining);
127         if (return_length + increment > max_length) {
128             break;
129         }
130         return_length += increment;
131         remaining -= increment;
132         iter->page++;
133 
134         if (iter->page > iter->last_page) {
135             break;
136         }
137 
138         zx_paddr_t next = phys_addrs[iter->page];
139         if (phys + PAGE_SIZE != next) {
140             break;
141         }
142         phys = next;
143     }
144 
145     if (return_length > max_length) {
146         return_length = max_length;
147     }
148     phys_iter_increment(iter, return_length);
149     return return_length;
150 }