1 // Copyright 2017 The Fuchsia Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "fvm/sparse-reader.h"
6
7 #include <utility>
8
9 namespace fvm {
Create(fbl::unique_fd fd,fbl::unique_ptr<SparseReader> * out)10 zx_status_t SparseReader::Create(fbl::unique_fd fd, fbl::unique_ptr<SparseReader>* out) {
11 return SparseReader::CreateHelper(std::move(fd), true /* verbose */, out);
12 }
CreateSilent(fbl::unique_fd fd,fbl::unique_ptr<SparseReader> * out)13 zx_status_t SparseReader::CreateSilent(fbl::unique_fd fd, fbl::unique_ptr<SparseReader>* out) {
14 return SparseReader::CreateHelper(std::move(fd), false /* verbose */, out);
15 }
16
CreateHelper(fbl::unique_fd fd,bool verbose,fbl::unique_ptr<SparseReader> * out)17 zx_status_t SparseReader::CreateHelper(fbl::unique_fd fd, bool verbose,
18 fbl::unique_ptr<SparseReader>* out) {
19 fbl::AllocChecker ac;
20 fbl::unique_ptr<SparseReader> reader(new (&ac) SparseReader(std::move(fd), verbose));
21 if (!ac.check()) {
22 return ZX_ERR_NO_MEMORY;
23 }
24
25 zx_status_t status;
26 if ((status = reader->ReadMetadata()) != ZX_OK) {
27 return status;
28 }
29
30 *out = std::move(reader);
31 return ZX_OK;
32 }
33
SparseReader(fbl::unique_fd fd,bool verbose)34 SparseReader::SparseReader(fbl::unique_fd fd, bool verbose) : compressed_(false), verbose_(verbose),
35 fd_(std::move(fd)) {}
36
ReadMetadata()37 zx_status_t SparseReader::ReadMetadata() {
38 // Read sparse image header.
39 fvm::sparse_image_t image;
40 if (read(fd_.get(), &image, sizeof(fvm::sparse_image_t)) != sizeof(fvm::sparse_image_t)) {
41 fprintf(stderr, "failed to read the sparse header\n");
42 return ZX_ERR_IO;
43 }
44
45 // Verify the header.
46 if (image.magic != fvm::kSparseFormatMagic) {
47 fprintf(stderr, "SparseReader: Bad magic\n");
48 return ZX_ERR_BAD_STATE;
49 } else if (image.version != fvm::kSparseFormatVersion) {
50 fprintf(stderr, "SparseReader: Unexpected sparse file version\n");
51 return ZX_ERR_BAD_STATE;
52 }
53
54 fbl::AllocChecker ac;
55 metadata_.reset(new (&ac) uint8_t[image.header_length]);
56 if (!ac.check()) {
57 return ZX_ERR_NO_MEMORY;
58 }
59
60 memcpy(metadata_.get(), &image, sizeof(image));
61
62 // Read remainder of metadata.
63 size_t off = sizeof(image);
64 while (off < image.header_length) {
65 ssize_t r = read(fd_.get(), &metadata_[off], image.header_length - off);
66 if (r < 0) {
67 fprintf(stderr, "SparseReader: Failed to read metadata\n");
68 return ZX_ERR_IO;
69 }
70 off += r;
71 }
72
73 // If image is compressed, additional setup is required
74 if (image.flags & fvm::kSparseFlagLz4) {
75 if (verbose_) {
76 printf("Found compressed file\n");
77 }
78
79 compressed_ = true;
80 // Initialize decompression context
81 LZ4F_errorCode_t errc = LZ4F_createDecompressionContext(&dctx_, LZ4F_VERSION);
82 if (LZ4F_isError(errc)) {
83 fprintf(stderr, "SparseReader: could not initialize decompression: %s\n",
84 LZ4F_getErrorName(errc));
85 return ZX_ERR_INTERNAL;
86 }
87
88 size_t src_sz = 4;
89 size_t dst_sz = 0;
90 fbl::unique_ptr<uint8_t[]> inbufptr(new (&ac) uint8_t[src_sz]);
91 if (!ac.check()) {
92 return ZX_ERR_NO_MEMORY;
93 }
94
95 uint8_t* inbuf = inbufptr.get();
96
97 // Read first 4 bytes to let LZ4 tell us how much it expects in the first pass.
98 ssize_t nr = read(fd_.get(), inbuf, src_sz);
99 if (nr < static_cast<ssize_t>(src_sz)) {
100 fprintf(stderr, "SparseReader: could not read from input\n");
101 return ZX_ERR_IO;
102 }
103
104 // Run decompress once to find out how much data we should read for the next decompress run
105 // Since we are not yet decompressing any actual data, the dst_buffer is null
106 to_read_ = LZ4F_decompress(dctx_, nullptr, &dst_sz, inbuf, &src_sz, NULL);
107 if (LZ4F_isError(to_read_)) {
108 fprintf(stderr, "SparseReader: could not decompress header: %s\n",
109 LZ4F_getErrorName(to_read_));
110 return ZX_ERR_INTERNAL;
111 }
112
113 if (to_read_ > LZ4_MAX_BLOCK_SIZE) {
114 to_read_ = LZ4_MAX_BLOCK_SIZE;
115 }
116
117 // Initialize data buffers
118 zx_status_t status;
119 if ((status = InitializeBuffer(LZ4_MAX_BLOCK_SIZE, &out_buf_)) != ZX_OK) {
120 return status;
121 } else if ((status = InitializeBuffer(LZ4_MAX_BLOCK_SIZE, &in_buf_)) != ZX_OK) {
122 return status;
123 }
124 }
125
126 return ZX_OK;
127 }
128
InitializeBuffer(size_t size,buffer_t * out_buf)129 zx_status_t SparseReader::InitializeBuffer(size_t size, buffer_t* out_buf) {
130 if (size < LZ4_MAX_BLOCK_SIZE) {
131 fprintf(stderr, "Buffer size must be >= %d\n", LZ4_MAX_BLOCK_SIZE);
132 return ZX_ERR_INVALID_ARGS;
133 }
134
135 out_buf->max_size = size;
136 out_buf->size = 0;
137 out_buf->offset = 0;
138 fbl::AllocChecker ac;
139 out_buf->data.reset(new (&ac) uint8_t[size]);
140
141 if (!ac.check()) {
142 return ZX_ERR_NO_MEMORY;
143 }
144
145 return ZX_OK;
146 }
147
~SparseReader()148 SparseReader::~SparseReader() {
149 PrintStats();
150
151 if (compressed_) {
152 LZ4F_freeDecompressionContext(dctx_);
153 }
154 }
155
Image()156 fvm::sparse_image_t* SparseReader::Image() {
157 return reinterpret_cast<fvm::sparse_image_t*>(metadata_.get());
158 }
159
Partitions()160 fvm::partition_descriptor_t* SparseReader::Partitions() {
161 return reinterpret_cast<fvm::partition_descriptor_t*>(
162 reinterpret_cast<uintptr_t>(metadata_.get()) +
163 sizeof(fvm::sparse_image_t));
164 }
165
ReadData(uint8_t * data,size_t length,size_t * actual)166 zx_status_t SparseReader::ReadData(uint8_t* data, size_t length, size_t* actual) {
167 #ifdef __Fuchsia__
168 zx_ticks_t start = zx_ticks_get();
169 #endif
170 size_t total_size = 0;
171 if (compressed_) {
172 if (out_buf_.is_empty() && to_read_ == 0) {
173 // There is no more to read
174 return ZX_ERR_OUT_OF_RANGE;
175 }
176
177 // Read previously decompressed data from buffer if possible
178 out_buf_.read(data, length, &total_size);
179
180 // If we still have data to read, start decompression (reading more from fd as needed)
181 while (total_size < length && to_read_ > 0) {
182 // Make sure data to read does not exceed max, and both buffers are empty
183 ZX_ASSERT(out_buf_.is_empty());
184 ZX_ASSERT(in_buf_.is_empty());
185 ZX_ASSERT(to_read_ <= in_buf_.max_size);
186
187 // Read specified amount from fd
188 zx_status_t status;
189 if ((status = ReadRaw(in_buf_.data.get(), to_read_, &in_buf_.size)) != ZX_OK) {
190 return status;
191 }
192
193 size_t src_sz = in_buf_.size;
194 size_t next = 0;
195
196 // Decompress all compressed data
197 while (in_buf_.offset < to_read_) {
198 size_t dst_sz = out_buf_.max_size - out_buf_.size;
199 next = LZ4F_decompress(dctx_, out_buf_.data.get() + out_buf_.size, &dst_sz,
200 in_buf_.data.get() + in_buf_.offset, &src_sz, NULL);
201 if (LZ4F_isError(next)) {
202 fprintf(stderr, "could not decompress input: %s\n", LZ4F_getErrorName(next));
203 return -1;
204 }
205
206 out_buf_.size += dst_sz;
207 in_buf_.offset += src_sz;
208 in_buf_.size -= src_sz;
209 src_sz = to_read_ - in_buf_.offset;
210 }
211
212 // Make sure we have read all data from in_buf_
213 if (in_buf_.size > 0) {
214 return ZX_ERR_IO;
215 }
216
217 in_buf_.offset = 0;
218
219 // Copy newly decompressed data from outbuf
220 size_t cp = fbl::min(length - total_size, static_cast<size_t>(out_buf_.size));
221 out_buf_.read(data + total_size, cp, &cp);
222 total_size += cp;
223 to_read_ = next;
224
225 if (to_read_ > LZ4_MAX_BLOCK_SIZE) {
226 to_read_ = LZ4_MAX_BLOCK_SIZE;
227 }
228 }
229 } else {
230 zx_status_t status = ReadRaw(data, length, &total_size);
231
232 if (status != ZX_OK) {
233 return status;
234 }
235 }
236
237 #ifdef __Fuchsia__
238 total_time_ += zx_ticks_get() - start;
239 #endif
240 *actual = total_size;
241 return ZX_OK;
242 }
243
ReadRaw(uint8_t * data,size_t length,size_t * actual)244 zx_status_t SparseReader::ReadRaw(uint8_t* data, size_t length, size_t* actual) {
245 #ifdef __Fuchsia__
246 zx_ticks_t start = zx_ticks_get();
247 #endif
248 ssize_t r;
249 size_t total_size = 0;
250 size_t bytes_left = length;
251 while ((r = read(fd_.get(), data + total_size, bytes_left)) > 0) {
252 total_size += r;
253 bytes_left -= r;
254 if (bytes_left == 0) {
255 break;
256 }
257 }
258
259 #ifdef __Fuchsia__
260 read_time_ += zx_ticks_get() - start;
261 #endif
262
263 if (r < 0) {
264 return static_cast<zx_status_t>(r);
265 }
266
267 *actual = total_size;
268 return ZX_OK;
269 }
270
WriteDecompressed(fbl::unique_fd outfd)271 zx_status_t SparseReader::WriteDecompressed(fbl::unique_fd outfd) {
272 if (!compressed_) {
273 fprintf(stderr, "BlockReader: File is not compressed\n");
274 return ZX_ERR_INVALID_ARGS;
275 }
276
277 // Update metadata and write to new file.
278 fvm::sparse_image_t* image = Image();
279 image->flags &= ~fvm::kSparseFlagLz4;
280
281 if (write(outfd.get(), metadata_.get(), image->header_length)
282 != static_cast<ssize_t>(image->header_length)) {
283 fprintf(stderr, "BlockReader: could not write header to out file\n");
284 return -1;
285 }
286
287 // Read/write decompressed data in LZ4_MAX_BLOCK_SIZE chunks.
288 while (true) {
289 zx_status_t status;
290 uint8_t data[LZ4_MAX_BLOCK_SIZE];
291 size_t length;
292 if ((status = ReadData(data, LZ4_MAX_BLOCK_SIZE, &length)) != ZX_OK) {
293 if (status == ZX_ERR_OUT_OF_RANGE) {
294 return ZX_OK;
295 }
296
297 return status;
298 }
299
300 if (write(outfd.get(), data, length) != static_cast<ssize_t>(length)) {
301 fprintf(stderr, "BlockReader: failed to write to output\n");
302 return ZX_ERR_IO;
303 }
304 }
305 }
306
PrintStats() const307 void SparseReader::PrintStats() const {
308 if (verbose_) {
309 printf("Reading FVM from compressed file: %s\n", compressed_ ? "true" : "false");
310 printf("Remaining bytes read into compression buffer: %lu\n", in_buf_.size);
311 printf("Remaining bytes written to decompression buffer: %lu\n", out_buf_.size);
312 #ifdef __Fuchsia__
313 printf("Time reading bytes from sparse FVM file: %lu (%lu s)\n", read_time_,
314 read_time_ / zx_ticks_per_second());
315 printf("Time reading bytes AND decompressing them: %lu (%lu s)\n", total_time_,
316 total_time_ / zx_ticks_per_second());
317 #endif
318 }
319 }
320
321 } // namespace fvm
322