1// Copyright 2015 The BoringSSL Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//go:build ignore
16
17package main
18
19import (
20	"bytes"
21	"errors"
22	"flag"
23	"fmt"
24	"os"
25	"os/exec"
26	"path"
27	"runtime"
28	"strconv"
29	"strings"
30	"sync"
31	"syscall"
32
33	"boringssl.googlesource.com/boringssl.git/util/testconfig"
34	"boringssl.googlesource.com/boringssl.git/util/testresult"
35)
36
37// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
38
39var (
40	useValgrind     = flag.Bool("valgrind", false, "If true, run code under valgrind")
41	useCallgrind    = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.")
42	useGDB          = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
43	useSDE          = flag.Bool("sde", false, "If true, run BoringSSL code under Intel's SDE for each supported chip")
44	sdePath         = flag.String("sde-path", "sde", "The path to find the sde binary.")
45	buildDir        = flag.String("build-dir", "build", "The build directory to run the tests from.")
46	numWorkers      = flag.Int("num-workers", runtime.NumCPU(), "Runs the given number of workers when testing.")
47	jsonOutput      = flag.String("json-output", "", "The file to output JSON results to.")
48	mallocTest      = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
49	mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
50	simulateARMCPUs = flag.Bool("simulate-arm-cpus", simulateARMCPUsDefault(), "If true, runs tests simulating different ARM CPUs.")
51	qemuBinary      = flag.String("qemu", "", "Optional, absolute path to a binary location for QEMU runtime.")
52)
53
54func simulateARMCPUsDefault() bool {
55	return (runtime.GOOS == "linux" || runtime.GOOS == "android") && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64")
56}
57
58type test struct {
59	testconfig.Test
60
61	shard, numShards int
62	// cpu, if not empty, contains a code to simulate. For SDE, run `sde64
63	// -help` to get a list of these codes. For ARM, see gtest_main.cc for
64	// the supported values.
65	cpu string
66}
67
68type result struct {
69	Test   test
70	Passed bool
71	Error  error
72}
73
74// sdeCPUs contains a list of CPU code that we run all tests under when *useSDE
75// is true.
76var sdeCPUs = []string{
77	"p4p", // Pentium4 Prescott
78	"mrm", // Merom
79	"pnr", // Penryn
80	"nhm", // Nehalem
81	"wsm", // Westmere
82	"snb", // Sandy Bridge
83	"ivb", // Ivy Bridge
84	"hsw", // Haswell
85	"bdw", // Broadwell
86	"slt", // Saltwell
87	"slm", // Silvermont
88	"glm", // Goldmont
89	"glp", // Goldmont Plus
90	"tnt", // Tremont
91	"skl", // Skylake
92	"cnl", // Cannon Lake
93	"icl", // Ice Lake
94	"skx", // Skylake server
95	"clx", // Cascade Lake
96	"cpx", // Cooper Lake
97	"icx", // Ice Lake server
98	"tgl", // Tiger Lake
99	"adl", // Alder Lake
100	"spr", // Sapphire Rapids
101}
102
103var armCPUs = []string{
104	"none",   // No support for any ARM extensions.
105	"neon",   // Support for NEON.
106	"crypto", // Support for NEON and crypto extensions.
107}
108
109func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
110	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"}
111	if dbAttach {
112		valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
113	}
114	valgrindArgs = append(valgrindArgs, path)
115	valgrindArgs = append(valgrindArgs, args...)
116
117	return exec.Command("valgrind", valgrindArgs...)
118}
119
120func callgrindOf(path string, args ...string) *exec.Cmd {
121	valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"}
122	valgrindArgs = append(valgrindArgs, path)
123	valgrindArgs = append(valgrindArgs, args...)
124
125	return exec.Command("valgrind", valgrindArgs...)
126}
127
128func gdbOf(path string, args ...string) *exec.Cmd {
129	xtermArgs := []string{"-e", "gdb", "--args"}
130	xtermArgs = append(xtermArgs, path)
131	xtermArgs = append(xtermArgs, args...)
132
133	return exec.Command("xterm", xtermArgs...)
134}
135
136func sdeOf(cpu, path string, args ...string) *exec.Cmd {
137	sdeArgs := []string{"-" + cpu}
138	// The kernel's vdso code for gettimeofday sometimes uses the RDTSCP
139	// instruction. Although SDE has a -chip_check_vsyscall flag that
140	// excludes such code by default, it does not seem to work. Instead,
141	// pass the -chip_check_exe_only flag which retains test coverage when
142	// statically linked and excludes the vdso.
143	if cpu == "p4p" || cpu == "pnr" || cpu == "mrm" || cpu == "slt" {
144		sdeArgs = append(sdeArgs, "-chip_check_exe_only")
145	}
146	sdeArgs = append(sdeArgs, "--", path)
147	sdeArgs = append(sdeArgs, args...)
148	return exec.Command(*sdePath, sdeArgs...)
149}
150
151func qemuOf(path string, args ...string) *exec.Cmd {
152	// The QEMU binary becomes the program to run, and the previous test program
153	// to run instead becomes an additional argument to the QEMU binary.
154	args = append([]string{path}, args...)
155	return exec.Command(*qemuBinary, args...)
156}
157
158var (
159	errMoreMallocs = errors.New("child process did not exhaust all allocation calls")
160	errTestSkipped = errors.New("test was skipped")
161)
162
163func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
164	prog := path.Join(*buildDir, test.Cmd[0])
165	args := append([]string{}, test.Cmd[1:]...)
166	if *simulateARMCPUs && test.cpu != "" {
167		args = append(args, "--cpu="+test.cpu)
168	}
169	if *useSDE {
170		// SDE is neither compatible with the unwind tester nor automatically
171		// detected.
172		args = append(args, "--no_unwind_tests")
173	}
174
175	var cmd *exec.Cmd
176	if *useValgrind {
177		cmd = valgrindOf(false, prog, args...)
178	} else if *useCallgrind {
179		cmd = callgrindOf(prog, args...)
180	} else if *useGDB {
181		cmd = gdbOf(prog, args...)
182	} else if *useSDE {
183		cmd = sdeOf(test.cpu, prog, args...)
184	} else if *qemuBinary != "" {
185		cmd = qemuOf(prog, args...)
186	} else {
187		cmd = exec.Command(prog, args...)
188	}
189	if test.Env != nil || test.numShards != 0 {
190		cmd.Env = make([]string, len(os.Environ()))
191		copy(cmd.Env, os.Environ())
192	}
193	if test.Env != nil {
194		cmd.Env = append(cmd.Env, test.Env...)
195	}
196	if test.numShards != 0 {
197		cmd.Env = append(cmd.Env, fmt.Sprintf("GTEST_SHARD_INDEX=%d", test.shard))
198		cmd.Env = append(cmd.Env, fmt.Sprintf("GTEST_TOTAL_SHARDS=%d", test.numShards))
199	}
200	var outBuf bytes.Buffer
201	cmd.Stdout = &outBuf
202	cmd.Stderr = &outBuf
203	if mallocNumToFail >= 0 {
204		cmd.Env = os.Environ()
205		cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
206		if *mallocTestDebug {
207			cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1")
208		}
209		cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1")
210	}
211
212	if err := cmd.Start(); err != nil {
213		return false, err
214	}
215	if err := cmd.Wait(); err != nil {
216		if exitError, ok := err.(*exec.ExitError); ok {
217			switch exitError.Sys().(syscall.WaitStatus).ExitStatus() {
218			case 88:
219				return false, errMoreMallocs
220			case 89:
221				fmt.Print(string(outBuf.Bytes()))
222				return false, errTestSkipped
223			}
224		}
225		fmt.Print(string(outBuf.Bytes()))
226		return false, err
227	}
228
229	// Account for Windows line-endings.
230	stdout := bytes.Replace(outBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
231
232	if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
233		(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
234		return true, nil
235	}
236
237	// Also accept a googletest-style pass line. This is left here in
238	// transition until the tests are all converted and this script made
239	// unnecessary.
240	if bytes.Contains(stdout, []byte("\n[  PASSED  ]")) {
241		return true, nil
242	}
243
244	fmt.Print(string(outBuf.Bytes()))
245	return false, nil
246}
247
248func runTest(test test) (bool, error) {
249	if *mallocTest < 0 {
250		return runTestOnce(test, -1)
251	}
252
253	for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ {
254		if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs {
255			if err != nil {
256				err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err)
257			}
258			return passed, err
259		}
260	}
261}
262
263// setWorkingDirectory walks up directories as needed until the current working
264// directory is the top of a BoringSSL checkout.
265func setWorkingDirectory() {
266	for i := 0; i < 64; i++ {
267		if _, err := os.Stat("BUILDING.md"); err == nil {
268			return
269		}
270		os.Chdir("..")
271	}
272
273	panic("Couldn't find BUILDING.md in a parent directory!")
274}
275
276func worker(tests <-chan test, results chan<- result, done *sync.WaitGroup) {
277	defer done.Done()
278	for test := range tests {
279		passed, err := runTest(test)
280		results <- result{test, passed, err}
281	}
282}
283
284func (t test) shortName() string {
285	return strings.Join(t.Cmd, " ") + t.shardMsg() + t.cpuMsg() + t.envMsg()
286}
287
288func SpaceIf(returnSpace bool) string {
289	if !returnSpace {
290		return ""
291	}
292	return " "
293}
294
295func (t test) longName() string {
296	return strings.Join(t.Env, " ") + SpaceIf(len(t.Env) != 0) + strings.Join(t.Cmd, " ") + t.shardMsg() + t.cpuMsg()
297}
298
299func (t test) shardMsg() string {
300	if t.numShards == 0 {
301		return ""
302	}
303
304	return fmt.Sprintf(" [shard %d/%d]", t.shard+1, t.numShards)
305}
306
307func (t test) cpuMsg() string {
308	if len(t.cpu) == 0 {
309		return ""
310	}
311
312	return fmt.Sprintf(" (for CPU %q)", t.cpu)
313}
314
315func (t test) envMsg() string {
316	if len(t.Env) == 0 {
317		return ""
318	}
319
320	return " (custom environment)"
321}
322
323func (t test) getGTestShards() ([]test, error) {
324	if *numWorkers == 1 || !t.Shard {
325		return []test{t}, nil
326	}
327
328	shards := make([]test, *numWorkers)
329	for i := range shards {
330		shards[i] = t
331		shards[i].shard = i
332		shards[i].numShards = *numWorkers
333	}
334
335	return shards, nil
336}
337
338func main() {
339	flag.Parse()
340	setWorkingDirectory()
341
342	testCases, err := testconfig.ParseTestConfig("util/all_tests.json")
343	if err != nil {
344		fmt.Printf("Failed to parse input: %s\n", err)
345		os.Exit(1)
346	}
347
348	var wg sync.WaitGroup
349	tests := make(chan test, *numWorkers)
350	results := make(chan result, *numWorkers)
351
352	for i := 0; i < *numWorkers; i++ {
353		wg.Add(1)
354		go worker(tests, results, &wg)
355	}
356
357	go func() {
358		for _, baseTest := range testCases {
359			test := test{Test: baseTest}
360			if *useSDE {
361				if test.SkipSDE {
362					continue
363				}
364				// SDE generates plenty of tasks and gets slower
365				// with additional sharding.
366				for _, cpu := range sdeCPUs {
367					testForCPU := test
368					testForCPU.cpu = cpu
369					tests <- testForCPU
370				}
371			} else if *simulateARMCPUs {
372				// This mode is run instead of the default path,
373				// so also include the native flow.
374				tests <- test
375				for _, cpu := range armCPUs {
376					testForCPU := test
377					testForCPU.cpu = cpu
378					tests <- testForCPU
379				}
380			} else {
381				shards, err := test.getGTestShards()
382				if err != nil {
383					fmt.Printf("Error listing tests: %s\n", err)
384					os.Exit(1)
385				}
386				for _, shard := range shards {
387					tests <- shard
388				}
389			}
390		}
391		close(tests)
392
393		wg.Wait()
394		close(results)
395	}()
396
397	testOutput := testresult.NewResults()
398	var failed, skipped []test
399	var total int
400	for testResult := range results {
401		test := testResult.Test
402		args := test.Cmd
403
404		total++
405		if testResult.Error == errTestSkipped {
406			fmt.Printf("%s\n", test.longName())
407			fmt.Printf("%s was skipped\n", args[0])
408			skipped = append(skipped, test)
409			testOutput.AddSkip(test.longName())
410		} else if testResult.Error != nil {
411			fmt.Printf("%s\n", test.longName())
412			fmt.Printf("%s failed to complete: %s\n", args[0], testResult.Error)
413			failed = append(failed, test)
414			testOutput.AddResult(test.longName(), "CRASH", testResult.Error)
415		} else if !testResult.Passed {
416			fmt.Printf("%s\n", test.longName())
417			fmt.Printf("%s failed to print PASS on the last line.\n", args[0])
418			failed = append(failed, test)
419			testOutput.AddResult(test.longName(), "FAIL", nil)
420		} else {
421			fmt.Printf("%s\n", test.shortName())
422			testOutput.AddResult(test.longName(), "PASS", nil)
423		}
424	}
425
426	if *jsonOutput != "" {
427		if err := testOutput.WriteToFile(*jsonOutput); err != nil {
428			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
429		}
430	}
431
432	if len(skipped) > 0 {
433		fmt.Printf("\n%d of %d tests were skipped:\n", len(skipped), total)
434		for _, test := range skipped {
435			fmt.Printf("\t%s\n", test.shortName())
436		}
437	}
438
439	if len(failed) > 0 {
440		fmt.Printf("\n%d of %d tests failed:\n", len(failed), total)
441		for _, test := range failed {
442			fmt.Printf("\t%s\n", test.shortName())
443		}
444		os.Exit(1)
445	}
446
447	fmt.Printf("All unit tests passed!\n")
448}
449