1 // Copyright 2018 The Fuchsia Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <dirent.h>
6 #include <errno.h>
7 #include <fcntl.h>
8 #include <libgen.h>
9 #include <limits.h>
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <sys/stat.h>
13 #include <unistd.h>
14
15 #include <fbl/auto_call.h>
16 #include <fbl/string_buffer.h>
17 #include <fbl/unique_fd.h>
18 #include <fbl/unique_ptr.h>
19 #include <fbl/vector.h>
20 #include <runtests-utils/runtests-utils.h>
21 #include <unittest/unittest.h>
22
23 #include "runtests-utils-test-globals.h"
24 #include "runtests-utils-test-utils.h"
25
26 namespace runtests {
27 namespace {
28
29 static constexpr char kEchoSuccessAndArgs[] = "echo Success! $@";
30 static constexpr char kEchoFailureAndArgs[] = "echo Failure! $@ 1>&2\nexit 77";
31
ParseTestNamesEmptyStr()32 bool ParseTestNamesEmptyStr() {
33 BEGIN_TEST;
34
35 fbl::String input("");
36 fbl::Vector<fbl::String> parsed;
37 ParseTestNames(input, &parsed);
38 EXPECT_EQ(0, parsed.size());
39
40 END_TEST;
41 }
42
ParseTestNamesEmptyStrInMiddle()43 bool ParseTestNamesEmptyStrInMiddle() {
44 BEGIN_TEST;
45
46 fbl::String input("a,,b");
47 fbl::Vector<fbl::String> parsed;
48 ParseTestNames(input, &parsed);
49 ASSERT_EQ(2, parsed.size());
50 EXPECT_STR_EQ("a", parsed[0].c_str());
51 EXPECT_STR_EQ("b", parsed[1].c_str());
52
53 END_TEST;
54 }
55
ParseTestNamesTrailingComma()56 bool ParseTestNamesTrailingComma() {
57 BEGIN_TEST;
58
59 fbl::String input("a,");
60 fbl::Vector<fbl::String> parsed;
61 ParseTestNames(input, &parsed);
62 ASSERT_EQ(1, parsed.size());
63 EXPECT_STR_EQ("a", parsed[0].c_str());
64
65 END_TEST;
66 }
67
ParseTestNamesNormal()68 bool ParseTestNamesNormal() {
69 BEGIN_TEST;
70
71 fbl::String input("a,b");
72 fbl::Vector<fbl::String> parsed;
73 ParseTestNames(input, &parsed);
74 ASSERT_EQ(2, parsed.size());
75 EXPECT_STR_EQ("a", parsed[0].c_str());
76 EXPECT_STR_EQ("b", parsed[1].c_str());
77
78 END_TEST;
79 }
80
EmptyWhitelist()81 bool EmptyWhitelist() {
82 BEGIN_TEST;
83
84 fbl::Vector<fbl::String> whitelist;
85 EXPECT_FALSE(IsInWhitelist("a", whitelist));
86
87 END_TEST;
88 }
89
NonemptyWhitelist()90 bool NonemptyWhitelist() {
91 BEGIN_TEST;
92
93 fbl::Vector<fbl::String> whitelist = {"b", "a"};
94 EXPECT_TRUE(IsInWhitelist("a", whitelist));
95
96 END_TEST;
97 }
98
JoinPathNoTrailingSlash()99 bool JoinPathNoTrailingSlash() {
100 BEGIN_TEST;
101
102 EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b", "c/d").c_str());
103
104 END_TEST;
105 }
106
JoinPathTrailingSlash()107 bool JoinPathTrailingSlash() {
108 BEGIN_TEST;
109
110 EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b/", "c/d").c_str());
111
112 END_TEST;
113 }
114
JoinPathAbsoluteChild()115 bool JoinPathAbsoluteChild() {
116 BEGIN_TEST;
117
118 EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b/", "/c/d").c_str());
119
120 END_TEST;
121 }
122
MkDirAllTooLong()123 bool MkDirAllTooLong() {
124 BEGIN_TEST;
125
126 char too_long[PATH_MAX + 2];
127 memset(too_long, 'a', PATH_MAX + 1);
128 too_long[PATH_MAX + 1] = '\0';
129 EXPECT_EQ(ENAMETOOLONG, MkDirAll(too_long));
130
131 END_TEST;
132 }
MkDirAllAlreadyExists()133 bool MkDirAllAlreadyExists() {
134 BEGIN_TEST;
135
136 ScopedTestDir test_dir;
137 const fbl::String already = JoinPath(test_dir.path(), "already");
138 const fbl::String exists = JoinPath(already, "exists");
139 ASSERT_EQ(0, mkdir(already.c_str(), 0755));
140 ASSERT_EQ(0, mkdir(exists.c_str(), 0755));
141 EXPECT_EQ(0, MkDirAll(exists));
142
143 END_TEST;
144 }
MkDirAllParentAlreadyExists()145 bool MkDirAllParentAlreadyExists() {
146 BEGIN_TEST;
147
148 ScopedTestDir test_dir;
149 const fbl::String parent = JoinPath(test_dir.path(), "existing-parent");
150 const fbl::String child = JoinPath(parent, "child");
151 ASSERT_EQ(0, mkdir(parent.c_str(), 0755));
152 EXPECT_EQ(0, MkDirAll(child));
153 struct stat s;
154 EXPECT_EQ(0, stat(child.c_str(), &s));
155
156 END_TEST;
157 }
MkDirAllParentDoesNotExist()158 bool MkDirAllParentDoesNotExist() {
159 BEGIN_TEST;
160
161 ScopedTestDir test_dir;
162 const fbl::String parent = JoinPath(test_dir.path(), "not-existing-parent");
163 const fbl::String child = JoinPath(parent, "child");
164 struct stat s;
165 ASSERT_NE(0, stat(parent.c_str(), &s));
166 EXPECT_EQ(0, MkDirAll(child));
167 EXPECT_EQ(0, stat(child.c_str(), &s));
168
169 END_TEST;
170 }
171
WriteSummaryJSONSucceeds()172 bool WriteSummaryJSONSucceeds() {
173 BEGIN_TEST;
174
175 // TODO(IN-499): Use fmemopen instead of tmpfile.
176 FILE* output_file = tmpfile();
177 ASSERT_NONNULL(output_file);
178 fbl::Vector<fbl::unique_ptr<Result>> results;
179 results.push_back(fbl::make_unique<Result>("/a", SUCCESS, 0));
180 results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0));
181 ASSERT_EQ(0, WriteSummaryJSON(results, "output.txt", "/tmp/file_path",
182 output_file));
183 // We don't have a JSON parser in zircon right now, so just hard-code the
184 // expected output.
185 const char kExpectedJSONOutput[] = R"({
186 "tests": [
187 {
188 "name": "/a",
189 "output_file": "a/output.txt",
190 "result": "PASS"
191 },
192 {
193 "name": "b",
194 "output_file": "b/output.txt",
195 "result": "FAIL"
196 }
197 ],
198 "outputs": {
199 "syslog_file": "/tmp/file_path"
200 }
201 }
202 )";
203 EXPECT_TRUE(CompareFileContents(output_file, kExpectedJSONOutput));
204 fclose(output_file);
205
206 END_TEST;
207 }
208
WriteSummaryJSONSucceedsWithoutSyslogPath()209 bool WriteSummaryJSONSucceedsWithoutSyslogPath() {
210 BEGIN_TEST;
211
212 // TODO(IN-499): Use fmemopen instead of tmpfile.
213 FILE* output_file = tmpfile();
214 ASSERT_NONNULL(output_file);
215 fbl::Vector<fbl::unique_ptr<Result>> results;
216 results.push_back(fbl::make_unique<Result>("/a", SUCCESS, 0));
217 results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0));
218 ASSERT_EQ(0, WriteSummaryJSON(results, "output.txt", /*syslog_path=*/"",
219 output_file));
220 // With an empty syslog_path, we expect no values under "outputs" and
221 // "syslog_file" to be generated in the JSON output.
222 const char kExpectedJSONOutput[] = R"({
223 "tests": [
224 {
225 "name": "/a",
226 "output_file": "a/output.txt",
227 "result": "PASS"
228 },
229 {
230 "name": "b",
231 "output_file": "b/output.txt",
232 "result": "FAIL"
233 }
234 ]
235 }
236 )";
237
238 EXPECT_TRUE(CompareFileContents(output_file, kExpectedJSONOutput));
239 fclose(output_file);
240
241 END_TEST;
242 }
243
WriteSummaryJSONBadTestName()244 bool WriteSummaryJSONBadTestName() {
245 BEGIN_TEST;
246
247 // TODO(IN-499): Use fmemopen instead of tmpfile.
248 FILE* output_file = tmpfile();
249 ASSERT_NONNULL(output_file);
250 // A test name and output file consisting entirely of slashes should trigger
251 // an error.
252 fbl::Vector<fbl::unique_ptr<Result>> results;
253 results.push_back(fbl::make_unique<Result>("///", SUCCESS, 0));
254 results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0));
255 ASSERT_NE(0, WriteSummaryJSON(results, /*output_file_basename=*/"///",
256 /*syslog_path=*/"/", output_file));
257 fclose(output_file);
258
259 END_TEST;
260 }
261
ResolveGlobsNoMatches()262 bool ResolveGlobsNoMatches() {
263 BEGIN_TEST;
264
265 ScopedTestDir test_dir;
266 fbl::Vector<fbl::String> resolved;
267 fbl::String test_fs_glob = JoinPath(test_dir.path(), "bar*");
268 const fbl::Vector<fbl::String> globs = {"/foo/bar/*", test_fs_glob};
269 ASSERT_EQ(0, ResolveGlobs(globs, &resolved));
270 EXPECT_EQ(0, resolved.size());
271
272 END_TEST;
273 }
274
ResolveGlobsMultipleMatches()275 bool ResolveGlobsMultipleMatches() {
276 BEGIN_TEST;
277
278 ScopedTestDir test_dir;
279 fbl::String existing_dir_path =
280 JoinPath(test_dir.path(), "existing-dir/prefix-suffix");
281 fbl::String existing_file_path = JoinPath(test_dir.path(), "existing-file");
282 fbl::String existing_dir_glob =
283 JoinPath(test_dir.path(), "existing-dir/prefix*");
284 const fbl::Vector<fbl::String> globs = {
285 "/does/not/exist/*",
286 existing_dir_glob, // matches existing_dir_path.
287 existing_file_path};
288 ASSERT_EQ(0, MkDirAll(existing_dir_path));
289 const int existing_file_fd = open(existing_file_path.c_str(), O_CREAT);
290 ASSERT_NE(-1, existing_file_fd, strerror(errno));
291 ASSERT_NE(-1, close(existing_file_fd), strerror(errno));
292 fbl::Vector<fbl::String> resolved;
293 ASSERT_EQ(0, ResolveGlobs(globs, &resolved));
294 ASSERT_EQ(2, resolved.size());
295 EXPECT_STR_EQ(existing_dir_path.c_str(), resolved[0].c_str());
296
297 END_TEST;
298 }
299
RunTestSuccess()300 bool RunTestSuccess() {
301 BEGIN_TEST;
302
303 ScopedTestDir test_dir;
304 fbl::String test_name = JoinPath(test_dir.path(), "succeed.sh");
305 const char* argv[] = {test_name.c_str(), nullptr};
306 ScopedScriptFile script(argv[0], "exit 0");
307 fbl::unique_ptr<Result> result = PlatformRunTest(argv, nullptr, nullptr);
308 EXPECT_STR_EQ(argv[0], result->name.c_str());
309 EXPECT_EQ(SUCCESS, result->launch_status);
310 EXPECT_EQ(0, result->return_code);
311
312 END_TEST;
313 }
314
RunTestSuccessWithStdout()315 bool RunTestSuccessWithStdout() {
316 BEGIN_TEST;
317
318 ScopedTestDir test_dir;
319 fbl::String test_name = JoinPath(test_dir.path(), "succeed.sh");
320 const char* argv[] = {test_name.c_str(), nullptr};
321 const char expected_output[] = "Expect this!\n";
322 // Produces expected_output, b/c echo adds newline
323 const char script_contents[] = "echo Expect this!";
324 ScopedScriptFile script(argv[0], script_contents);
325
326 fbl::String output_filename = JoinPath(test_dir.path(), "test.out");
327 fbl::unique_ptr<Result> result =
328 PlatformRunTest(argv, nullptr, output_filename.c_str());
329
330 FILE* output_file = fopen(output_filename.c_str(), "r");
331 ASSERT_TRUE(output_file);
332 char buf[1024];
333 memset(buf, 0, sizeof(buf));
334 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
335 fclose(output_file);
336 EXPECT_STR_EQ(expected_output, buf);
337 EXPECT_STR_EQ(argv[0], result->name.c_str());
338 EXPECT_EQ(SUCCESS, result->launch_status);
339 EXPECT_EQ(0, result->return_code);
340
341 END_TEST;
342 }
343
RunTestFailureWithStderr()344 bool RunTestFailureWithStderr() {
345 BEGIN_TEST;
346
347 ScopedTestDir test_dir;
348 fbl::String test_name = JoinPath(test_dir.path(), "fail.sh");
349 const char* argv[] = {test_name.c_str(), nullptr};
350 const char expected_output[] = "Expect this!\n";
351 // Produces expected_output, b/c echo adds newline
352 const char script_contents[] = "echo Expect this! 1>&2\nexit 77";
353 ScopedScriptFile script(argv[0], script_contents);
354
355 fbl::String output_filename = JoinPath(test_dir.path(), "test.out");
356 fbl::unique_ptr<Result> result =
357 PlatformRunTest(argv, nullptr, output_filename.c_str());
358
359 FILE* output_file = fopen(output_filename.c_str(), "r");
360 ASSERT_TRUE(output_file);
361 char buf[1024];
362 memset(buf, 0, sizeof(buf));
363 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
364 fclose(output_file);
365 EXPECT_STR_EQ(expected_output, buf);
366 EXPECT_STR_EQ(argv[0], result->name.c_str());
367 EXPECT_EQ(FAILED_NONZERO_RETURN_CODE, result->launch_status);
368 EXPECT_EQ(77, result->return_code);
369
370 END_TEST;
371 }
372
RunTestFailureToLoadFile()373 bool RunTestFailureToLoadFile() {
374 BEGIN_TEST;
375
376 const char* argv[] = {"i/do/not/exist/", nullptr};
377
378 fbl::unique_ptr<Result> result = PlatformRunTest(argv, nullptr, nullptr);
379 EXPECT_STR_EQ(argv[0], result->name.c_str());
380 EXPECT_EQ(FAILED_TO_LAUNCH, result->launch_status);
381
382 END_TEST;
383 }
384
DiscoverTestsInDirGlobsBasic()385 bool DiscoverTestsInDirGlobsBasic() {
386 BEGIN_TEST;
387
388 ScopedTestDir test_dir;
389 const fbl::String a_file_name = JoinPath(test_dir.path(), "a.sh");
390 ScopedScriptFile a_file(a_file_name, "");
391 const fbl::String b_file_name = JoinPath(test_dir.path(), "b.sh");
392 ScopedScriptFile b_file(b_file_name, "");
393 fbl::Vector<fbl::String> discovered_paths;
394 EXPECT_EQ(0, DiscoverTestsInDirGlobs({test_dir.path()}, nullptr, {},
395 &discovered_paths));
396 EXPECT_EQ(2, discovered_paths.size());
397 bool discovered_a = false;
398 bool discovered_b = false;
399 // The order of the results is not defined, so just check that each is
400 // present.
401 for (const auto& path : discovered_paths) {
402 if (fbl::StringPiece(path) == a_file.path()) {
403 discovered_a = true;
404 } else if (fbl::StringPiece(path) == b_file.path()) {
405 discovered_b = true;
406 }
407 }
408 EXPECT_TRUE(discovered_a);
409 EXPECT_TRUE(discovered_b);
410
411 END_TEST;
412 }
413
DiscoverTestsInDirGlobsFilter()414 bool DiscoverTestsInDirGlobsFilter() {
415 BEGIN_TEST;
416
417 ScopedTestDir test_dir;
418 const char kHopefullyUniqueFileBasename[] =
419 "e829cea9919fe045ca199945db7ac99a";
420 const fbl::String unique_file_name =
421 JoinPath(test_dir.path(), kHopefullyUniqueFileBasename);
422 ScopedScriptFile unique_file(unique_file_name, "");
423 // This one should be ignored because its basename is not in the white list.
424 const fbl::String other_file_name = JoinPath(test_dir.path(), "foo.sh");
425 ScopedScriptFile fail_file(other_file_name, "");
426 fbl::Vector<fbl::String> discovered_paths;
427 EXPECT_EQ(0, DiscoverTestsInDirGlobs({JoinPath(TestFsRoot(), "*")}, nullptr,
428 {kHopefullyUniqueFileBasename},
429 &discovered_paths));
430 EXPECT_EQ(1, discovered_paths.size());
431 EXPECT_STR_EQ(unique_file_name.c_str(), discovered_paths[0].c_str());
432
433 END_TEST;
434 }
435
DiscoverTestsInDirGlobsIgnore()436 bool DiscoverTestsInDirGlobsIgnore() {
437 BEGIN_TEST;
438 ScopedTestDir test_dir_a, test_dir_b;
439 const fbl::String a_name = JoinPath(test_dir_a.path(), "foo.sh");
440 ScopedScriptFile a_file(a_name, "");
441 const fbl::String b_name = JoinPath(test_dir_b.path(), "foo.sh");
442 ScopedScriptFile fail_file(b_name, "");
443 fbl::Vector<fbl::String> discovered_paths;
444 EXPECT_EQ(0, DiscoverTestsInDirGlobs({test_dir_a.path(), test_dir_b.path()},
445 test_dir_b.basename(), {},
446 &discovered_paths));
447 EXPECT_EQ(1, discovered_paths.size());
448 EXPECT_STR_EQ(a_name.c_str(), discovered_paths[0].c_str());
449 END_TEST;
450 }
451
DiscoverTestsInListFileWithTrailingWhitespace()452 bool DiscoverTestsInListFileWithTrailingWhitespace() {
453 BEGIN_TEST;
454 // TODO(IN-499): Use fmemopen instead of tmpfile.
455 FILE* test_list_file = tmpfile();
456 ASSERT_NONNULL(test_list_file);
457 fprintf(test_list_file, "trailing/tab\t\n");
458 fprintf(test_list_file, "trailing/space \n");
459 fprintf(test_list_file, "trailing/return\r");
460 rewind(test_list_file);
461 fbl::Vector<fbl::String> test_paths;
462 EXPECT_EQ(0, DiscoverTestsInListFile(test_list_file, &test_paths));
463 EXPECT_EQ(3, test_paths.size());
464 EXPECT_STR_EQ("trailing/tab", test_paths[0].c_str());
465 EXPECT_STR_EQ("trailing/space", test_paths[1].c_str());
466 EXPECT_STR_EQ("trailing/return", test_paths[2].c_str());
467 fclose(test_list_file);
468 END_TEST;
469 }
470
RunTestsWithVerbosity()471 bool RunTestsWithVerbosity() {
472 BEGIN_TEST;
473
474 ScopedTestDir test_dir;
475 const fbl::String succeed_file_name =
476 JoinPath(test_dir.path(), "succeed.sh");
477 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
478 int num_failed = 0;
479 fbl::Vector<fbl::unique_ptr<Result>> results;
480 const signed char verbosity = 77;
481 const fbl::String output_dir = JoinPath(test_dir.path(), "output");
482 const char output_file_base_name[] = "output.txt";
483 ASSERT_EQ(0, MkDirAll(output_dir));
484 EXPECT_TRUE(RunTests(PlatformRunTest, {succeed_file_name}, {},
485 output_dir.c_str(), output_file_base_name, verbosity,
486 &num_failed, &results));
487 EXPECT_EQ(0, num_failed);
488 EXPECT_EQ(1, results.size());
489
490 fbl::String output_path = JoinPath(
491 JoinPath(output_dir, succeed_file.path()), output_file_base_name);
492 FILE* output_file = fopen(output_path.c_str(), "r");
493 ASSERT_TRUE(output_file);
494 char buf[1024];
495 memset(buf, 0, sizeof(buf));
496 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
497 fclose(output_file);
498 EXPECT_STR_EQ("Success! v=77\n", buf);
499
500 END_TEST;
501 }
502
RunTestsWithArguments()503 bool RunTestsWithArguments() {
504 BEGIN_TEST;
505
506 ScopedTestDir test_dir;
507 const fbl::String succeed_file_name =
508 JoinPath(test_dir.path(), "succeed.sh");
509 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
510 int num_failed = 0;
511 const signed char verbosity = -1;
512 fbl::Vector<fbl::unique_ptr<Result>> results;
513 fbl::Vector<fbl::String> args{"first", "second", "third", "-4", "--", "-", "seventh"};
514 const fbl::String output_dir = JoinPath(test_dir.path(), "output");
515 const char output_file_base_name[] = "output.txt";
516 ASSERT_EQ(0, MkDirAll(output_dir));
517 EXPECT_TRUE(RunTests(PlatformRunTest, {succeed_file_name}, args,
518 output_dir.c_str(), output_file_base_name, verbosity,
519 &num_failed, &results));
520 EXPECT_EQ(0, num_failed);
521 EXPECT_EQ(1, results.size());
522
523 fbl::String output_path = JoinPath(
524 JoinPath(output_dir, succeed_file.path()), output_file_base_name);
525 FILE* output_file = fopen(output_path.c_str(), "r");
526 ASSERT_TRUE(output_file);
527 char buf[1024];
528 memset(buf, 0, sizeof(buf));
529 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
530 fclose(output_file);
531 EXPECT_STR_EQ("Success! first second third -4 -- - seventh\n", buf);
532
533 END_TEST;
534 }
535
DiscoverAndRunTestsBasicPass()536 bool DiscoverAndRunTestsBasicPass() {
537 BEGIN_TEST;
538
539 ScopedTestDir test_dir;
540 const fbl::String succeed_file_name1 =
541 JoinPath(test_dir.path(), "succeed1.sh");
542 ScopedScriptFile succeed_file1(succeed_file_name1, kEchoSuccessAndArgs);
543 const fbl::String succeed_file_name2 =
544 JoinPath(test_dir.path(), "succeed2.sh");
545 ScopedScriptFile succeed_file2(succeed_file_name2, kEchoSuccessAndArgs);
546 const char* const argv[] = {"./runtests", test_dir.path()};
547 TestStopwatch stopwatch;
548 EXPECT_EQ(EXIT_SUCCESS, DiscoverAndRunTests(PlatformRunTest, 2, argv, {},
549 &stopwatch, ""));
550
551 END_TEST;
552 }
553
DiscoverAndRunTestsBasicFail()554 bool DiscoverAndRunTestsBasicFail() {
555 BEGIN_TEST;
556
557 ScopedTestDir test_dir;
558 const fbl::String succeed_file_name =
559 JoinPath(test_dir.path(), "succeed.sh");
560 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
561 const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh");
562 ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs);
563 const char* const argv[] = {"./runtests", test_dir.path()};
564 TestStopwatch stopwatch;
565 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 2, argv, {},
566 &stopwatch, ""));
567
568 END_TEST;
569 }
570
DiscoverAndRunTestsFallsBackToDefaultDirs()571 bool DiscoverAndRunTestsFallsBackToDefaultDirs() {
572 BEGIN_TEST;
573
574 ScopedTestDir test_dir;
575 const fbl::String succeed_file_name =
576 JoinPath(test_dir.path(), "succeed.sh");
577 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
578 const char* const argv[] = {"./runtests"};
579 TestStopwatch stopwatch;
580 EXPECT_EQ(EXIT_SUCCESS,
581 DiscoverAndRunTests(PlatformRunTest, 1, argv, {test_dir.path()},
582 &stopwatch, ""));
583
584 END_TEST;
585 }
586
DiscoverAndRunTestsFailsWithNoTestGlobsOrDefaultDirs()587 bool DiscoverAndRunTestsFailsWithNoTestGlobsOrDefaultDirs() {
588 BEGIN_TEST;
589
590 ScopedTestDir test_dir;
591 const fbl::String succeed_file_name =
592 JoinPath(test_dir.path(), "succeed.sh");
593 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
594 const char* const argv[] = {"./runtests"};
595 TestStopwatch stopwatch;
596 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 1, argv, {},
597 &stopwatch, ""));
598
599 END_TEST;
600 }
601
DiscoverAndRunTestsFailsWithBadArgs()602 bool DiscoverAndRunTestsFailsWithBadArgs() {
603 BEGIN_TEST;
604
605 ScopedTestDir test_dir;
606 const fbl::String succeed_file_name =
607 JoinPath(test_dir.path(), "succeed.sh");
608 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
609 const char* const argv[] = {"./runtests", "-?", "unknown-arg",
610 test_dir.path()};
611 TestStopwatch stopwatch;
612 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {},
613 &stopwatch, ""));
614
615 END_TEST;
616 }
617
DiscoverAndRunTestsWithGlobs()618 bool DiscoverAndRunTestsWithGlobs() {
619 BEGIN_TEST;
620
621 ScopedTestDir test_dir;
622 // Make the directories that the following globs will match.
623 const fbl::String dir1 = JoinPath(test_dir.path(), "A/B/C");
624 EXPECT_EQ(0, MkDirAll(dir1));
625 const fbl::String dir2 = JoinPath(test_dir.path(), "A/D/C");
626 EXPECT_EQ(0, MkDirAll(dir2));
627
628 const fbl::String succeed_file_name1 =
629 JoinPath(test_dir.path(), "succeed.sh");
630 ScopedScriptFile succeed_file1(succeed_file_name1, kEchoSuccessAndArgs);
631 const fbl::String succeed_file_name2 = JoinPath(dir1, "succeed.sh");
632 ScopedScriptFile succeed_file2(succeed_file_name2, kEchoSuccessAndArgs);
633 const fbl::String succeed_file_name3 = JoinPath(dir2, "succeed.sh");
634 ScopedScriptFile succeed_file3(succeed_file_name3, kEchoSuccessAndArgs);
635
636 fbl::String glob = JoinPath(test_dir.path(), "A/*/C");
637 const char* const argv[] = {"./runtests", test_dir.path(), glob.c_str()};
638 TestStopwatch stopwatch;
639 EXPECT_EQ(EXIT_SUCCESS, DiscoverAndRunTests(PlatformRunTest, 3, argv, {},
640 &stopwatch, ""));
641
642 END_TEST;
643 }
644
645 // Passing an -o argument should result in output being written to that
646 // location.
DiscoverAndRunTestsWithOutput()647 bool DiscoverAndRunTestsWithOutput() {
648 BEGIN_TEST;
649
650 ScopedTestDir test_dir;
651 const fbl::String succeed_file_name =
652 JoinPath(test_dir.path(), "succeed.sh");
653 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
654 const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh");
655 ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs);
656
657 const fbl::String output_dir =
658 JoinPath(test_dir.path(), "run-all-tests-output-1");
659 EXPECT_EQ(0, MkDirAll(output_dir));
660
661 const char* const argv[] = {"./runtests", "-o", output_dir.c_str(),
662 test_dir.path()};
663 TestStopwatch stopwatch;
664 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {},
665 &stopwatch, ""));
666
667 // Prepare the expected output.
668 fbl::String success_output_rel_path;
669 ASSERT_TRUE(GetOutputFileRelPath(output_dir, succeed_file_name,
670 &success_output_rel_path));
671 fbl::String failure_output_rel_path;
672 ASSERT_TRUE(GetOutputFileRelPath(output_dir, fail_file_name,
673 &failure_output_rel_path));
674
675 fbl::StringBuffer<1024> expected_pass_output_buf;
676 expected_pass_output_buf.AppendPrintf(
677 " {\n"
678 " \"name\": \"%s\",\n"
679 " \"output_file\": \"%s\",\n"
680 " \"result\": \"PASS\"\n"
681 " }",
682 succeed_file_name.c_str(),
683 success_output_rel_path.c_str() +
684 1); // +1 to discard the leading slash.
685 fbl::StringBuffer<1024> expected_fail_output_buf;
686 expected_fail_output_buf.AppendPrintf(
687 " {\n"
688 " \"name\": \"%s\",\n"
689 " \"output_file\": \"%s\",\n"
690 " \"result\": \"FAIL\"\n"
691 " }",
692 fail_file_name.c_str(),
693 failure_output_rel_path.c_str() +
694 1); // +1 to discared the leading slash.
695
696 // Extract the actual output.
697 const fbl::String output_path = JoinPath(output_dir, "summary.json");
698 FILE* output_file = fopen(output_path.c_str(), "r");
699 ASSERT_TRUE(output_file);
700 char buf[1024];
701 memset(buf, 0, sizeof(buf));
702 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
703 fclose(output_file);
704
705 // The order of the tests in summary.json is not defined, so first check the
706 // prefix, then be permissive about order of the actual tests.
707 size_t buf_index = 0;
708 EXPECT_EQ(0, strncmp(kExpectedJSONOutputPrefix, &buf[buf_index],
709 kExpectedJSONOutputPrefixSize));
710 buf_index += kExpectedJSONOutputPrefixSize;
711
712 if (!strncmp(expected_pass_output_buf.c_str(), &buf[buf_index],
713 expected_pass_output_buf.size())) {
714 buf_index += expected_pass_output_buf.size();
715 EXPECT_EQ(0, strncmp(",\n", &buf[buf_index], sizeof(",\n") - 1));
716 buf_index += sizeof(",\n") - 1;
717 EXPECT_EQ(0, strncmp(expected_fail_output_buf.c_str(), &buf[buf_index],
718 expected_fail_output_buf.size()));
719 buf_index += expected_fail_output_buf.size();
720 } else if (!strncmp(expected_fail_output_buf.c_str(), &buf[buf_index],
721 expected_fail_output_buf.size())) {
722 buf_index += expected_fail_output_buf.size();
723 EXPECT_EQ(0, strncmp(",\n", &buf[buf_index], sizeof(",\n") - 1));
724 buf_index += sizeof(",\n") - 1;
725 EXPECT_EQ(0, strncmp(expected_pass_output_buf.c_str(), &buf[buf_index],
726 expected_pass_output_buf.size()));
727 buf_index += expected_pass_output_buf.size();
728 } else {
729 printf("Unexpected buffer contents: %s\n", buf);
730 EXPECT_TRUE(false,
731 "output buf didn't contain expected pass or fail strings");
732 }
733 EXPECT_STR_EQ("\n ]\n}\n", &buf[buf_index]);
734
735 END_TEST;
736 }
737
738 // Passing an -o argument *and* a syslog file name should result in output being
739 // written that includes a syslog reference.
DiscoverAndRunTestsWithSyslogOutput()740 bool DiscoverAndRunTestsWithSyslogOutput() {
741 BEGIN_TEST;
742
743 ScopedTestDir test_dir;
744 const fbl::String succeed_file_name =
745 JoinPath(test_dir.path(), "succeed.sh");
746 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
747 const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh");
748 ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs);
749
750 const fbl::String output_dir =
751 JoinPath(test_dir.path(), "run-all-tests-output-2");
752 EXPECT_EQ(0, MkDirAll(output_dir));
753
754 const char* const argv[] = {"./runtests", "-o", output_dir.c_str(),
755 test_dir.path()};
756 TestStopwatch stopwatch;
757 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {},
758 &stopwatch, "syslog.txt"));
759
760 // Prepare the expected output.
761 fbl::String success_output_rel_path;
762 ASSERT_TRUE(GetOutputFileRelPath(output_dir, succeed_file_name,
763 &success_output_rel_path));
764 fbl::String failure_output_rel_path;
765 ASSERT_TRUE(GetOutputFileRelPath(output_dir, fail_file_name,
766 &failure_output_rel_path));
767
768 const char kExpectedOutputsStr[] =
769 " \"outputs\": {\n"
770 " \"syslog_file\": \"syslog.txt\"\n"
771 " }";
772
773 // Extract the actual output.
774 const fbl::String output_path = JoinPath(output_dir, "summary.json");
775 FILE* output_file = fopen(output_path.c_str(), "r");
776 ASSERT_TRUE(output_file);
777 char buf[1024];
778 memset(buf, 0, sizeof(buf));
779 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
780 fclose(output_file);
781
782 // We don't actually care if the string is at the beginning or the end of
783 // the JSON, so just search for it anywhere.
784 bool found_expected_outputs_str = false;
785 for (size_t buf_index = 0; buf[buf_index]; ++buf_index) {
786 if (!strncmp(kExpectedOutputsStr, &buf[buf_index],
787 sizeof(kExpectedOutputsStr) - 1)) {
788 found_expected_outputs_str = true;
789 break;
790 }
791 }
792 if (!found_expected_outputs_str) {
793 printf("Unexpected buffer contents: %s\n", buf);
794 }
795 EXPECT_TRUE(found_expected_outputs_str,
796 "Didn't find expected outputs str in buf");
797
798 END_TEST;
799 }
800
801 BEGIN_TEST_CASE(ParseTestNames)
802 RUN_TEST(ParseTestNamesEmptyStr)
803 RUN_TEST(ParseTestNamesEmptyStrInMiddle)
804 RUN_TEST(ParseTestNamesNormal)
805 RUN_TEST(ParseTestNamesTrailingComma)
806 END_TEST_CASE(ParseTestNames)
807
808 BEGIN_TEST_CASE(IsInWhitelist)
809 RUN_TEST(EmptyWhitelist)
810 RUN_TEST(NonemptyWhitelist)
811 END_TEST_CASE(IsInWhitelist)
812
813 BEGIN_TEST_CASE(JoinPath)
814 RUN_TEST(JoinPathNoTrailingSlash)
815 RUN_TEST(JoinPathTrailingSlash)
816 RUN_TEST(JoinPathAbsoluteChild)
817 END_TEST_CASE(JoinPath)
818
819 BEGIN_TEST_CASE(MkDirAll)
820 RUN_TEST(MkDirAllTooLong)
821 RUN_TEST(MkDirAllAlreadyExists)
822 RUN_TEST(MkDirAllParentAlreadyExists)
823 RUN_TEST(MkDirAllParentDoesNotExist)
824 END_TEST_CASE(MkDirAll)
825
826 BEGIN_TEST_CASE(WriteSummaryJSON)
827 RUN_TEST_MEDIUM(WriteSummaryJSONSucceeds)
828 RUN_TEST_MEDIUM(WriteSummaryJSONSucceedsWithoutSyslogPath)
829 RUN_TEST_MEDIUM(WriteSummaryJSONBadTestName)
830 END_TEST_CASE(WriteSummaryJSON)
831
832 BEGIN_TEST_CASE(ResolveGlobs)
833 RUN_TEST(ResolveGlobsNoMatches)
834 RUN_TEST(ResolveGlobsMultipleMatches)
835 END_TEST_CASE(ResolveGlobs)
836
837 BEGIN_TEST_CASE(RunTest)
838 RUN_TEST(RunTestSuccess)
839 RUN_TEST(RunTestSuccessWithStdout)
840 RUN_TEST(RunTestFailureWithStderr)
841 RUN_TEST(RunTestFailureToLoadFile)
842 END_TEST_CASE(RunTest)
843
844 BEGIN_TEST_CASE(DiscoverTestsInDirGlobs)
845 RUN_TEST(DiscoverTestsInDirGlobsBasic)
846 RUN_TEST(DiscoverTestsInDirGlobsFilter)
847 RUN_TEST(DiscoverTestsInDirGlobsIgnore)
848 END_TEST_CASE(DiscoverTestsInDirGlobs)
849
850 BEGIN_TEST_CASE(DiscoverTestsInListFile)
851 RUN_TEST(DiscoverTestsInListFileWithTrailingWhitespace)
852 END_TEST_CASE(DiscoverTestsInListFile)
853
854 BEGIN_TEST_CASE(RunTests)
855 RUN_TEST_MEDIUM(RunTestsWithVerbosity)
856 RUN_TEST_MEDIUM(RunTestsWithArguments)
857 END_TEST_CASE(RunTests)
858
859 BEGIN_TEST_CASE(DiscoverAndRunTests)
860 RUN_TEST_MEDIUM(DiscoverAndRunTestsBasicPass)
861 RUN_TEST_MEDIUM(DiscoverAndRunTestsBasicFail)
862 RUN_TEST_MEDIUM(DiscoverAndRunTestsFallsBackToDefaultDirs)
863 RUN_TEST_MEDIUM(DiscoverAndRunTestsFailsWithNoTestGlobsOrDefaultDirs)
864 RUN_TEST_MEDIUM(DiscoverAndRunTestsFailsWithBadArgs)
865 RUN_TEST_MEDIUM(DiscoverAndRunTestsWithGlobs)
866 RUN_TEST_MEDIUM(DiscoverAndRunTestsWithOutput)
867 RUN_TEST_MEDIUM(DiscoverAndRunTestsWithSyslogOutput)
868 END_TEST_CASE(DiscoverAndRunTests)
869 } // namespace
870 } // namespace runtests
871