1 // Copyright 2018 The Fuchsia Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // Helper functions for running test binaries and recording their results. 6 7 #ifndef ZIRCON_SYSTEM_ULIB_RUNTESTS_UTILS_INCLUDE_RUNTESTS_UTILS_RUNTESTS_UTILS_H_ 8 #define ZIRCON_SYSTEM_ULIB_RUNTESTS_UTILS_INCLUDE_RUNTESTS_UTILS_RUNTESTS_UTILS_H_ 9 10 #include <inttypes.h> 11 12 #include <fbl/intrusive_hash_table.h> 13 #include <fbl/intrusive_single_list.h> 14 #include <fbl/string.h> 15 #include <fbl/string_piece.h> 16 #include <fbl/unique_ptr.h> 17 #include <fbl/vector.h> 18 #include <lib/zircon-internal/fnv1hash.h> 19 #include <zircon/types.h> 20 21 namespace runtests { 22 23 // Status of launching a test subprocess. 24 enum LaunchStatus { 25 SUCCESS, 26 FAILED_TO_LAUNCH, 27 FAILED_TO_WAIT, 28 FAILED_DURING_IO, 29 FAILED_TO_RETURN_CODE, 30 FAILED_NONZERO_RETURN_CODE, 31 FAILED_COLLECTING_SINK_DATA, 32 FAILED_UNKNOWN, 33 }; 34 35 // Represents a single dumpfile element. 36 struct DumpFile { 37 fbl::String name; // Name of the dumpfile. 38 fbl::String file; // File name for the content. 39 }; 40 41 // Represents data published through data sink. 42 struct DataSink : public fbl::SinglyLinkedListable<fbl::unique_ptr<DataSink>> { 43 fbl::String name; // Name of the data sink. 44 fbl::Vector<DumpFile> files; // All the sink dumpfiles. 45 DataSinkDataSink46 explicit DataSink(fbl::String name) 47 : name(name) {} 48 49 // Runtimes may publish more than one file with the same data sink name. We use hash table 50 // that's mapping data sink name to the list of file names to store these efficiently. GetKeyDataSink51 fbl::String GetKey() const { return name; } GetHashDataSink52 static size_t GetHash(fbl::String key) { return fnv1a64str(key.c_str()); } 53 }; 54 55 // Represents the result of a single test run. 56 struct Result { 57 fbl::String name; // argv[0]. 58 LaunchStatus launch_status; 59 int64_t return_code; // Only valid if launch_status == SUCCESS or FAILED_NONZERO_RETURN_CODE. 60 using HashTable = fbl::HashTable<fbl::String, fbl::unique_ptr<DataSink>>; 61 HashTable data_sinks; // Mapping from data sink name to list of files. 62 // TODO(ZX-2050): Track duration of test binary. 63 64 // Constructor really only needed until we have C++14, which will allow call-sites to use 65 // aggregate initializer syntax. ResultResult66 Result(const char* name_arg, LaunchStatus launch_status_arg, int64_t return_code_arg) 67 : name(name_arg), launch_status(launch_status_arg), return_code(return_code_arg) {} 68 }; 69 70 // Function that invokes a test binary and writes its output to a file. 71 // 72 // |argv| is the commandline to use to run the test program; must be 73 // null-terminated. 74 // |output_dir| is the output directory for test's data sinks. May be nullptr, in which case 75 // no data sinks will be saved. 76 // |output_filename| is the name of the file to which the test binary's output 77 // will be written. May be nullptr, in which case the output will not be 78 // redirected. 79 typedef fbl::unique_ptr<Result> (*RunTestFn)(const char* argv[], 80 const char* output_dir, 81 const char* output_filename); 82 83 // A means of measuring how long it takes to run tests. 84 class Stopwatch { 85 public: 86 virtual ~Stopwatch() = default; 87 88 // Starts timing. 89 virtual void Start() = 0; 90 91 // Returns the elapsed time in milliseconds since invoking Start(), or else 92 // since initialization if Start() has not yet been called. 93 virtual int64_t DurationInMsecs() = 0; 94 }; 95 96 // Splits |input| by ',' and appends the results onto |output|. 97 // Empty strings are not put into output. 98 void ParseTestNames(fbl::StringPiece input, fbl::Vector<fbl::String>* output); 99 100 // Returns true iff |name| is equal to one of strings in |whitelist|. 101 bool IsInWhitelist(fbl::StringPiece name, const fbl::Vector<fbl::String>& whitelist); 102 103 // Ensures |dir_name| exists by creating it and its parents if it doesn't. 104 // Returns 0 on success, else an error code compatible with errno. 105 int MkDirAll(fbl::StringPiece dir_name); 106 107 // Returns "|parent|/|child|". Unless child is absolute, in which case it returns |child|. 108 // 109 // |parent| is the parent path. 110 // |child| is the child path. 111 fbl::String JoinPath(fbl::StringPiece parent, fbl::StringPiece child); 112 113 // Writes a JSON summary of test results given a sequence of results. 114 // 115 // |results| are the run results to summarize. 116 // |output_file_basename| is base name of output file. 117 // |syslog_path| is the file path where syslogs are written. 118 // |summary_json| is the file stream to write the JSON summary to. 119 // 120 // Returns 0 on success, else an error code compatible with errno. 121 int WriteSummaryJSON(const fbl::Vector<fbl::unique_ptr<Result>>& results, 122 fbl::StringPiece output_file_basename, 123 fbl::StringPiece syslog_path, 124 FILE* summary_json); 125 126 // Resolves a set of globs. 127 // 128 // |globs| is an array of glob patterns. 129 // |resolved| will hold the results of resolving |globs|. 130 // 131 // Returns 0 on success, else an error code from glob.h. 132 int ResolveGlobs(const fbl::Vector<fbl::String>& globs, 133 fbl::Vector<fbl::String>* resolved); 134 135 // Executes all specified binaries. 136 // 137 // |run_test| is the function used to invoke the test binaries. 138 // |test_paths| are the paths of the binaries to execute. 139 // |test_args| are arguments passed into the binaries under test. 140 // |output_dir| is the output directory for all the tests' output. May be nullptr, in which case 141 // output will not be captured. 142 // |output_file_basename| is the basename of the tests' output files. May be nullptr only if 143 // |output_dir| is also nullptr. 144 // Each test's standard output and standard error will be written to 145 // |output_dir|/<test binary path>/|output_file_basename|. 146 // |verbosity| if > 0 is converted to a string and passed as an additional argument to the 147 // tests, so argv = {test_path, "v=<verbosity>"}. Also if > 0, this function prints more output 148 // to stdout than otherwise. 149 // |num_failed| is an output parameter which will be set to the number of test 150 // binaries that failed. 151 // |results| is an output parameter to which run results will be appended. 152 // 153 // Returns false if any test binary failed, true otherwise. 154 bool RunTests(const RunTestFn& RunTest, const fbl::Vector<fbl::String>& test_paths, 155 const fbl::Vector<fbl::String>& test_args, 156 const char* output_dir, const fbl::StringPiece output_file_basename, 157 signed char verbosity, int* failed_count, 158 fbl::Vector<fbl::unique_ptr<Result>>* results); 159 160 // Expands |dir_globs| and searches those directories for files. 161 // 162 // |dir_globs| are expanded as globs to directory names, and then those directories are searched. 163 // |ignore_dir_name| iff not null, any directory with this basename will not be searched. 164 // |basename_whitelist| iff not empty, only files that have a basename in this whitelist will be 165 // returned. 166 // |test_paths| is an output parameter to which absolute file paths will be appended. 167 // 168 // Returns 0 on success, else an error code compatible with errno. 169 int DiscoverTestsInDirGlobs(const fbl::Vector<fbl::String>& dir_globs, const char* ignore_dir_name, 170 const fbl::Vector<fbl::String>& basename_whitelist, 171 fbl::Vector<fbl::String>* test_paths); 172 173 // Reads |test_list_file| and appends whatever tests it finds to |test_paths|. 174 // 175 // Returns 0 on success, else an error code compatible with errno. 176 int DiscoverTestsInListFile(FILE* test_list_file, fbl::Vector<fbl::String>* test_paths); 177 178 // Discovers and runs tests based on command line arguments. 179 // 180 // |RunTest|: function to run each test. 181 // |argc|: length of |argv|. 182 // |argv|: see //system/ulib/runtests-utils/discover-and-run-tests.cpp, 183 // specifically the 'Usage()' function, for documentation. 184 // |default_test_dirs|: directories in which to look for tests if no test 185 // directory globs are specified. 186 // |stopwatch|: for timing how long all tests took to run. 187 // |syslog_file_name|: if an output directory is specified ("-o"), syslog ouput 188 // will be written to a file under that directory and this name. 189 // 190 // Returns EXIT_SUCCESS if all tests passed; else, returns EXIT_FAILURE. 191 int DiscoverAndRunTests(const RunTestFn& RunTest, int argc, const char* const* argv, 192 const fbl::Vector<fbl::String>& default_test_dirs, 193 Stopwatch* stopwatch, const fbl::StringPiece syslog_file_name); 194 195 } // namespace runtests 196 197 #endif // ZIRCON_SYSTEM_ULIB_RUNTESTS_UTILS_INCLUDE_RUNTESTS_UTILS_RUNTESTS_UTILS_H_ 198