1 /*
2  * ECMA Test 262 Runner for QuickJS
3  *
4  * Copyright (c) 2017-2020 Fabrice Bellard
5  * Copyright (c) 2017-2020 Charlie Gordon
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a copy
8  * of this software and associated documentation files (the "Software"), to deal
9  * in the Software without restriction, including without limitation the rights
10  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11  * copies of the Software, and to permit persons to whom the Software is
12  * furnished to do so, subject to the following conditions:
13  *
14  * The above copyright notice and this permission notice shall be included in
15  * all copies or substantial portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23  * THE SOFTWARE.
24  */
25 #include <stdlib.h>
26 #include <stdio.h>
27 #include <stdarg.h>
28 #include <inttypes.h>
29 #include <string.h>
30 #include <assert.h>
31 #include <ctype.h>
32 #include <unistd.h>
33 #include <errno.h>
34 #include <time.h>
35 #include <dirent.h>
36 #include <ftw.h>
37 
38 #include "cutils.h"
39 #include "list.h"
40 #include "quickjs-libc.h"
41 
42 /* enable test262 thread support to test SharedArrayBuffer and Atomics */
43 #define CONFIG_AGENT
44 
45 #define CMD_NAME "run-test262"
46 
47 typedef struct namelist_t {
48     char **array;
49     int count;
50     int size;
51     unsigned int sorted : 1;
52 } namelist_t;
53 
54 namelist_t test_list;
55 namelist_t exclude_list;
56 namelist_t exclude_dir_list;
57 
58 FILE *outfile;
59 enum test_mode_t {
60     TEST_DEFAULT_NOSTRICT, /* run tests as nostrict unless test is flagged as strictonly */
61     TEST_DEFAULT_STRICT,   /* run tests as strict unless test is flagged as nostrict */
62     TEST_NOSTRICT,         /* run tests as nostrict, skip strictonly tests */
63     TEST_STRICT,           /* run tests as strict, skip nostrict tests */
64     TEST_ALL,              /* run tests in both strict and nostrict, unless restricted by spec */
65 } test_mode = TEST_DEFAULT_NOSTRICT;
66 int skip_async;
67 int skip_module;
68 int new_style;
69 int dump_memory;
70 int stats_count;
71 JSMemoryUsage stats_all, stats_avg, stats_min, stats_max;
72 char *stats_min_filename;
73 char *stats_max_filename;
74 int verbose;
75 char *harness_dir;
76 char *harness_exclude;
77 char *harness_features;
78 char *harness_skip_features;
79 char *error_filename;
80 char *error_file;
81 FILE *error_out;
82 char *report_filename;
83 int update_errors;
84 int test_count, test_failed, test_index, test_skipped, test_excluded;
85 int new_errors, changed_errors, fixed_errors;
86 int async_done;
87 
88 void warning(const char *, ...) __attribute__((__format__(__printf__, 1, 2)));
89 void fatal(int, const char *, ...) __attribute__((__format__(__printf__, 2, 3)));
90 
warning(const char * fmt,...)91 void warning(const char *fmt, ...)
92 {
93     va_list ap;
94 
95     fflush(stdout);
96     fprintf(stderr, "%s: ", CMD_NAME);
97     va_start(ap, fmt);
98     vfprintf(stderr, fmt, ap);
99     va_end(ap);
100     fputc('\n', stderr);
101 }
102 
fatal(int errcode,const char * fmt,...)103 void fatal(int errcode, const char *fmt, ...)
104 {
105     va_list ap;
106 
107     fflush(stdout);
108     fprintf(stderr, "%s: ", CMD_NAME);
109     va_start(ap, fmt);
110     vfprintf(stderr, fmt, ap);
111     va_end(ap);
112     fputc('\n', stderr);
113 
114     exit(errcode);
115 }
116 
perror_exit(int errcode,const char * s)117 void perror_exit(int errcode, const char *s)
118 {
119     fflush(stdout);
120     fprintf(stderr, "%s: ", CMD_NAME);
121     perror(s);
122     exit(errcode);
123 }
124 
strdup_len(const char * str,int len)125 char *strdup_len(const char *str, int len)
126 {
127     char *p = malloc(len + 1);
128     memcpy(p, str, len);
129     p[len] = '\0';
130     return p;
131 }
132 
str_equal(const char * a,const char * b)133 static inline int str_equal(const char *a, const char *b) {
134     return !strcmp(a, b);
135 }
136 
str_append(char ** pp,const char * sep,const char * str)137 char *str_append(char **pp, const char *sep, const char *str) {
138     char *res, *p;
139     size_t len = 0;
140     p = *pp;
141     if (p) {
142         len = strlen(p) + strlen(sep);
143     }
144     res = malloc(len + strlen(str) + 1);
145     if (p) {
146         strcpy(res, p);
147         strcat(res, sep);
148     }
149     strcpy(res + len, str);
150     free(p);
151     return *pp = res;
152 }
153 
str_strip(char * p)154 char *str_strip(char *p)
155 {
156     size_t len = strlen(p);
157     while (len > 0 && isspace((unsigned char)p[len - 1]))
158         p[--len] = '\0';
159     while (isspace((unsigned char)*p))
160         p++;
161     return p;
162 }
163 
has_prefix(const char * str,const char * prefix)164 int has_prefix(const char *str, const char *prefix)
165 {
166     return !strncmp(str, prefix, strlen(prefix));
167 }
168 
skip_prefix(const char * str,const char * prefix)169 char *skip_prefix(const char *str, const char *prefix)
170 {
171     int i;
172     for (i = 0;; i++) {
173         if (prefix[i] == '\0') {  /* skip the prefix */
174             str += i;
175             break;
176         }
177         if (str[i] != prefix[i])
178             break;
179     }
180     return (char *)str;
181 }
182 
get_basename(const char * filename)183 char *get_basename(const char *filename)
184 {
185     char *p;
186 
187     p = strrchr(filename, '/');
188     if (!p)
189         return NULL;
190     return strdup_len(filename, p - filename);
191 }
192 
compose_path(const char * path,const char * name)193 char *compose_path(const char *path, const char *name)
194 {
195     int path_len, name_len;
196     char *d, *q;
197 
198     if (!path || path[0] == '\0' || *name == '/') {
199         d = strdup(name);
200     } else {
201         path_len = strlen(path);
202         name_len = strlen(name);
203         d = malloc(path_len + 1 + name_len + 1);
204         if (d) {
205             q = d;
206             memcpy(q, path, path_len);
207             q += path_len;
208             if (path[path_len - 1] != '/')
209                 *q++ = '/';
210             memcpy(q, name, name_len + 1);
211         }
212     }
213     return d;
214 }
215 
namelist_cmp(const char * a,const char * b)216 int namelist_cmp(const char *a, const char *b)
217 {
218     /* compare strings in modified lexicographical order */
219     for (;;) {
220         int ca = (unsigned char)*a++;
221         int cb = (unsigned char)*b++;
222         if (isdigit(ca) && isdigit(cb)) {
223             int na = ca - '0';
224             int nb = cb - '0';
225             while (isdigit(ca = (unsigned char)*a++))
226                 na = na * 10 + ca - '0';
227             while (isdigit(cb = (unsigned char)*b++))
228                 nb = nb * 10 + cb - '0';
229             if (na < nb)
230                 return -1;
231             if (na > nb)
232                 return +1;
233         }
234         if (ca < cb)
235             return -1;
236         if (ca > cb)
237             return +1;
238         if (ca == '\0')
239             return 0;
240     }
241 }
242 
namelist_cmp_indirect(const void * a,const void * b)243 int namelist_cmp_indirect(const void *a, const void *b)
244 {
245     return namelist_cmp(*(const char **)a, *(const char **)b);
246 }
247 
namelist_sort(namelist_t * lp)248 void namelist_sort(namelist_t *lp)
249 {
250     int i, count;
251     if (lp->count > 1) {
252         qsort(lp->array, lp->count, sizeof(*lp->array), namelist_cmp_indirect);
253         /* remove duplicates */
254         for (count = i = 1; i < lp->count; i++) {
255             if (namelist_cmp(lp->array[count - 1], lp->array[i]) == 0) {
256                 free(lp->array[i]);
257             } else {
258                 lp->array[count++] = lp->array[i];
259             }
260         }
261         lp->count = count;
262     }
263     lp->sorted = 1;
264 }
265 
namelist_find(namelist_t * lp,const char * name)266 int namelist_find(namelist_t *lp, const char *name)
267 {
268     int a, b, m, cmp;
269 
270     if (!lp->sorted) {
271         namelist_sort(lp);
272     }
273     for (a = 0, b = lp->count; a < b;) {
274         m = a + (b - a) / 2;
275         cmp = namelist_cmp(lp->array[m], name);
276         if (cmp < 0)
277             a = m + 1;
278         else if (cmp > 0)
279             b = m;
280         else
281             return m;
282     }
283     return -1;
284 }
285 
namelist_add(namelist_t * lp,const char * base,const char * name)286 void namelist_add(namelist_t *lp, const char *base, const char *name)
287 {
288     char *s;
289 
290     s = compose_path(base, name);
291     if (!s)
292         goto fail;
293     if (lp->count == lp->size) {
294         size_t newsize = lp->size + (lp->size >> 1) + 4;
295         char **a = realloc(lp->array, sizeof(lp->array[0]) * newsize);
296         if (!a)
297             goto fail;
298         lp->array = a;
299         lp->size = newsize;
300     }
301     lp->array[lp->count] = s;
302     lp->count++;
303     return;
304 fail:
305     fatal(1, "allocation failure\n");
306 }
307 
namelist_load(namelist_t * lp,const char * filename)308 void namelist_load(namelist_t *lp, const char *filename)
309 {
310     char buf[1024];
311     char *base_name;
312     FILE *f;
313 
314     f = fopen(filename, "rb");
315     if (!f) {
316         perror_exit(1, filename);
317     }
318     base_name = get_basename(filename);
319 
320     while (fgets(buf, sizeof(buf), f) != NULL) {
321         char *p = str_strip(buf);
322         if (*p == '#' || *p == ';' || *p == '\0')
323             continue;  /* line comment */
324 
325         namelist_add(lp, base_name, p);
326     }
327     free(base_name);
328     fclose(f);
329 }
330 
namelist_add_from_error_file(namelist_t * lp,const char * file)331 void namelist_add_from_error_file(namelist_t *lp, const char *file)
332 {
333     const char *p, *p0;
334     char *pp;
335 
336     for (p = file; (p = strstr(p, ".js:")) != NULL; p++) {
337         for (p0 = p; p0 > file && p0[-1] != '\n'; p0--)
338             continue;
339         pp = strdup_len(p0, p + 3 - p0);
340         namelist_add(lp, NULL, pp);
341         free(pp);
342     }
343 }
344 
namelist_free(namelist_t * lp)345 void namelist_free(namelist_t *lp)
346 {
347     while (lp->count > 0) {
348         free(lp->array[--lp->count]);
349     }
350     free(lp->array);
351     lp->array = NULL;
352     lp->size = 0;
353 }
354 
add_test_file(const char * filename,const struct stat * ptr,int flag)355 static int add_test_file(const char *filename, const struct stat *ptr, int flag)
356 {
357     namelist_t *lp = &test_list;
358     if (has_suffix(filename, ".js") && !has_suffix(filename, "_FIXTURE.js"))
359         namelist_add(lp, NULL, filename);
360     return 0;
361 }
362 
363 /* find js files from the directory tree and sort the list */
enumerate_tests(const char * path)364 static void enumerate_tests(const char *path)
365 {
366     namelist_t *lp = &test_list;
367     int start = lp->count;
368     ftw(path, add_test_file, 100);
369     qsort(lp->array + start, lp->count - start, sizeof(*lp->array),
370           namelist_cmp_indirect);
371 }
372 
js_print(JSContext * ctx,JSValueConst this_val,int argc,JSValueConst * argv)373 static JSValue js_print(JSContext *ctx, JSValueConst this_val,
374                         int argc, JSValueConst *argv)
375 {
376     int i;
377     const char *str;
378 
379     if (outfile) {
380         for (i = 0; i < argc; i++) {
381             if (i != 0)
382                 fputc(' ', outfile);
383             str = JS_ToCString(ctx, argv[i]);
384             if (!str)
385                 return JS_EXCEPTION;
386             if (!strcmp(str, "Test262:AsyncTestComplete")) {
387                 async_done++;
388             } else if (strstart(str, "Test262:AsyncTestFailure", NULL)) {
389                 async_done = 2; /* force an error */
390             }
391             fputs(str, outfile);
392             JS_FreeCString(ctx, str);
393         }
394         fputc('\n', outfile);
395     }
396     return JS_UNDEFINED;
397 }
398 
js_detachArrayBuffer(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)399 static JSValue js_detachArrayBuffer(JSContext *ctx, JSValue this_val,
400                                     int argc, JSValue *argv)
401 {
402     JS_DetachArrayBuffer(ctx, argv[0]);
403     return JS_UNDEFINED;
404 }
405 
js_evalScript(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)406 static JSValue js_evalScript(JSContext *ctx, JSValue this_val,
407                              int argc, JSValue *argv)
408 {
409     const char *str;
410     size_t len;
411     JSValue ret;
412     str = JS_ToCStringLen(ctx, &len, argv[0]);
413     if (!str)
414         return JS_EXCEPTION;
415     ret = JS_Eval(ctx, str, len, "<evalScript>", JS_EVAL_TYPE_GLOBAL);
416     JS_FreeCString(ctx, str);
417     return ret;
418 }
419 
420 #ifdef CONFIG_AGENT
421 
422 #include <pthread.h>
423 
424 typedef struct {
425     struct list_head link;
426     pthread_t tid;
427     char *script;
428     JSValue broadcast_func;
429     BOOL broadcast_pending;
430     JSValue broadcast_sab; /* in the main context */
431     uint8_t *broadcast_sab_buf;
432     size_t broadcast_sab_size;
433     int32_t broadcast_val;
434 } Test262Agent;
435 
436 typedef struct {
437     struct list_head link;
438     char *str;
439 } AgentReport;
440 
441 static JSValue add_helpers1(JSContext *ctx);
442 static void add_helpers(JSContext *ctx);
443 
444 static pthread_mutex_t agent_mutex = PTHREAD_MUTEX_INITIALIZER;
445 static pthread_cond_t agent_cond = PTHREAD_COND_INITIALIZER;
446 /* list of Test262Agent.link */
447 static struct list_head agent_list = LIST_HEAD_INIT(agent_list);
448 
449 static pthread_mutex_t report_mutex = PTHREAD_MUTEX_INITIALIZER;
450 /* list of AgentReport.link */
451 static struct list_head report_list = LIST_HEAD_INIT(report_list);
452 
agent_start(void * arg)453 static void *agent_start(void *arg)
454 {
455     Test262Agent *agent = arg;
456     JSRuntime *rt;
457     JSContext *ctx;
458     JSValue ret_val;
459     int ret;
460 
461     rt = JS_NewRuntime();
462     if (rt == NULL) {
463         fatal(1, "JS_NewRuntime failure");
464     }
465     ctx = JS_NewContext(rt);
466     if (ctx == NULL) {
467         JS_FreeRuntime(rt);
468         fatal(1, "JS_NewContext failure");
469     }
470     JS_SetContextOpaque(ctx, agent);
471     JS_SetRuntimeInfo(rt, "agent");
472     JS_SetCanBlock(rt, TRUE);
473 
474     add_helpers(ctx);
475     ret_val = JS_Eval(ctx, agent->script, strlen(agent->script),
476                       "<evalScript>", JS_EVAL_TYPE_GLOBAL);
477     free(agent->script);
478     agent->script = NULL;
479     if (JS_IsException(ret_val))
480         js_std_dump_error(ctx);
481     JS_FreeValue(ctx, ret_val);
482 
483     for(;;) {
484         JSContext *ctx1;
485         ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
486         if (ret < 0) {
487             js_std_dump_error(ctx);
488             break;
489         } else if (ret == 0) {
490             if (JS_IsUndefined(agent->broadcast_func)) {
491                 break;
492             } else {
493                 JSValue args[2];
494 
495                 pthread_mutex_lock(&agent_mutex);
496                 while (!agent->broadcast_pending) {
497                     pthread_cond_wait(&agent_cond, &agent_mutex);
498                 }
499 
500                 agent->broadcast_pending = FALSE;
501                 pthread_cond_signal(&agent_cond);
502 
503                 pthread_mutex_unlock(&agent_mutex);
504 
505                 args[0] = JS_NewArrayBuffer(ctx, agent->broadcast_sab_buf,
506                                             agent->broadcast_sab_size,
507                                             NULL, NULL, TRUE);
508                 args[1] = JS_NewInt32(ctx, agent->broadcast_val);
509                 ret_val = JS_Call(ctx, agent->broadcast_func, JS_UNDEFINED,
510                                   2, (JSValueConst *)args);
511                 JS_FreeValue(ctx, args[0]);
512                 JS_FreeValue(ctx, args[1]);
513                 if (JS_IsException(ret_val))
514                     js_std_dump_error(ctx);
515                 JS_FreeValue(ctx, ret_val);
516                 JS_FreeValue(ctx, agent->broadcast_func);
517                 agent->broadcast_func = JS_UNDEFINED;
518             }
519         }
520     }
521     JS_FreeValue(ctx, agent->broadcast_func);
522 
523     JS_FreeContext(ctx);
524     JS_FreeRuntime(rt);
525     return NULL;
526 }
527 
js_agent_start(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)528 static JSValue js_agent_start(JSContext *ctx, JSValue this_val,
529                               int argc, JSValue *argv)
530 {
531     const char *script;
532     Test262Agent *agent;
533 
534     if (JS_GetContextOpaque(ctx) != NULL)
535         return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
536 
537     script = JS_ToCString(ctx, argv[0]);
538     if (!script)
539         return JS_EXCEPTION;
540     agent = malloc(sizeof(*agent));
541     memset(agent, 0, sizeof(*agent));
542     agent->broadcast_func = JS_UNDEFINED;
543     agent->broadcast_sab = JS_UNDEFINED;
544     agent->script = strdup(script);
545     JS_FreeCString(ctx, script);
546     list_add_tail(&agent->link, &agent_list);
547     pthread_create(&agent->tid, NULL, agent_start, agent);
548     return JS_UNDEFINED;
549 }
550 
js_agent_free(JSContext * ctx)551 static void js_agent_free(JSContext *ctx)
552 {
553     struct list_head *el, *el1;
554     Test262Agent *agent;
555 
556     list_for_each_safe(el, el1, &agent_list) {
557         agent = list_entry(el, Test262Agent, link);
558         pthread_join(agent->tid, NULL);
559         JS_FreeValue(ctx, agent->broadcast_sab);
560         list_del(&agent->link);
561         free(agent);
562     }
563 }
564 
js_agent_leaving(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)565 static JSValue js_agent_leaving(JSContext *ctx, JSValue this_val,
566                                 int argc, JSValue *argv)
567 {
568     Test262Agent *agent = JS_GetContextOpaque(ctx);
569     if (!agent)
570         return JS_ThrowTypeError(ctx, "must be called inside an agent");
571     /* nothing to do */
572     return JS_UNDEFINED;
573 }
574 
is_broadcast_pending(void)575 static BOOL is_broadcast_pending(void)
576 {
577     struct list_head *el;
578     Test262Agent *agent;
579     list_for_each(el, &agent_list) {
580         agent = list_entry(el, Test262Agent, link);
581         if (agent->broadcast_pending)
582             return TRUE;
583     }
584     return FALSE;
585 }
586 
js_agent_broadcast(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)587 static JSValue js_agent_broadcast(JSContext *ctx, JSValue this_val,
588                                   int argc, JSValue *argv)
589 {
590     JSValueConst sab = argv[0];
591     struct list_head *el;
592     Test262Agent *agent;
593     uint8_t *buf;
594     size_t buf_size;
595     int32_t val;
596 
597     if (JS_GetContextOpaque(ctx) != NULL)
598         return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
599 
600     buf = JS_GetArrayBuffer(ctx, &buf_size, sab);
601     if (!buf)
602         return JS_EXCEPTION;
603     if (JS_ToInt32(ctx, &val, argv[1]))
604         return JS_EXCEPTION;
605 
606     /* broadcast the values and wait until all agents have started
607        calling their callbacks */
608     pthread_mutex_lock(&agent_mutex);
609     list_for_each(el, &agent_list) {
610         agent = list_entry(el, Test262Agent, link);
611         agent->broadcast_pending = TRUE;
612         /* the shared array buffer is used by the thread, so increment
613            its refcount */
614         agent->broadcast_sab = JS_DupValue(ctx, sab);
615         agent->broadcast_sab_buf = buf;
616         agent->broadcast_sab_size = buf_size;
617         agent->broadcast_val = val;
618     }
619     pthread_cond_broadcast(&agent_cond);
620 
621     while (is_broadcast_pending()) {
622         pthread_cond_wait(&agent_cond, &agent_mutex);
623     }
624     pthread_mutex_unlock(&agent_mutex);
625     return JS_UNDEFINED;
626 }
627 
js_agent_receiveBroadcast(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)628 static JSValue js_agent_receiveBroadcast(JSContext *ctx, JSValue this_val,
629                                          int argc, JSValue *argv)
630 {
631     Test262Agent *agent = JS_GetContextOpaque(ctx);
632     if (!agent)
633         return JS_ThrowTypeError(ctx, "must be called inside an agent");
634     if (!JS_IsFunction(ctx, argv[0]))
635         return JS_ThrowTypeError(ctx, "expecting function");
636     JS_FreeValue(ctx, agent->broadcast_func);
637     agent->broadcast_func = JS_DupValue(ctx, argv[0]);
638     return JS_UNDEFINED;
639 }
640 
js_agent_sleep(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)641 static JSValue js_agent_sleep(JSContext *ctx, JSValue this_val,
642                               int argc, JSValue *argv)
643 {
644     uint32_t duration;
645     if (JS_ToUint32(ctx, &duration, argv[0]))
646         return JS_EXCEPTION;
647     usleep(duration * 1000);
648     return JS_UNDEFINED;
649 }
650 
get_clock_ms(void)651 static int64_t get_clock_ms(void)
652 {
653     struct timespec ts;
654     clock_gettime(CLOCK_MONOTONIC, &ts);
655     return (uint64_t)ts.tv_sec * 1000 + (ts.tv_nsec / 1000000);
656 }
657 
js_agent_monotonicNow(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)658 static JSValue js_agent_monotonicNow(JSContext *ctx, JSValue this_val,
659                                      int argc, JSValue *argv)
660 {
661     return JS_NewInt64(ctx, get_clock_ms());
662 }
663 
js_agent_getReport(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)664 static JSValue js_agent_getReport(JSContext *ctx, JSValue this_val,
665                                   int argc, JSValue *argv)
666 {
667     AgentReport *rep;
668     JSValue ret;
669 
670     pthread_mutex_lock(&report_mutex);
671     if (list_empty(&report_list)) {
672         rep = NULL;
673     } else {
674         rep = list_entry(report_list.next, AgentReport, link);
675         list_del(&rep->link);
676     }
677     pthread_mutex_unlock(&report_mutex);
678     if (rep) {
679         ret = JS_NewString(ctx, rep->str);
680         free(rep->str);
681         free(rep);
682     } else {
683         ret = JS_NULL;
684     }
685     return ret;
686 }
687 
js_agent_report(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)688 static JSValue js_agent_report(JSContext *ctx, JSValue this_val,
689                                int argc, JSValue *argv)
690 {
691     const char *str;
692     AgentReport *rep;
693 
694     str = JS_ToCString(ctx, argv[0]);
695     if (!str)
696         return JS_EXCEPTION;
697     rep = malloc(sizeof(*rep));
698     rep->str = strdup(str);
699     JS_FreeCString(ctx, str);
700 
701     pthread_mutex_lock(&report_mutex);
702     list_add_tail(&rep->link, &report_list);
703     pthread_mutex_unlock(&report_mutex);
704     return JS_UNDEFINED;
705 }
706 
707 static const JSCFunctionListEntry js_agent_funcs[] = {
708     /* only in main */
709     JS_CFUNC_DEF("start", 1, js_agent_start ),
710     JS_CFUNC_DEF("getReport", 0, js_agent_getReport ),
711     JS_CFUNC_DEF("broadcast", 2, js_agent_broadcast ),
712     /* only in agent */
713     JS_CFUNC_DEF("report", 1, js_agent_report ),
714     JS_CFUNC_DEF("leaving", 0, js_agent_leaving ),
715     JS_CFUNC_DEF("receiveBroadcast", 1, js_agent_receiveBroadcast ),
716     /* in both */
717     JS_CFUNC_DEF("sleep", 1, js_agent_sleep ),
718     JS_CFUNC_DEF("monotonicNow", 0, js_agent_monotonicNow ),
719 };
720 
js_new_agent(JSContext * ctx)721 static JSValue js_new_agent(JSContext *ctx)
722 {
723     JSValue agent;
724     agent = JS_NewObject(ctx);
725     JS_SetPropertyFunctionList(ctx, agent, js_agent_funcs,
726                                countof(js_agent_funcs));
727     return agent;
728 }
729 #endif
730 
js_createRealm(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)731 static JSValue js_createRealm(JSContext *ctx, JSValue this_val,
732                               int argc, JSValue *argv)
733 {
734     JSContext *ctx1;
735     JSValue ret;
736 
737     ctx1 = JS_NewContext(JS_GetRuntime(ctx));
738     if (!ctx1)
739         return JS_ThrowOutOfMemory(ctx);
740     ret = add_helpers1(ctx1);
741     /* ctx1 has a refcount so it stays alive */
742     JS_FreeContext(ctx1);
743     return ret;
744 }
745 
add_helpers1(JSContext * ctx)746 static JSValue add_helpers1(JSContext *ctx)
747 {
748     JSValue global_obj;
749     JSValue obj262;
750 
751     global_obj = JS_GetGlobalObject(ctx);
752 
753     JS_SetPropertyStr(ctx, global_obj, "print",
754                       JS_NewCFunction(ctx, js_print, "print", 1));
755 
756     /* $262 special object used by the tests */
757     obj262 = JS_NewObject(ctx);
758     JS_SetPropertyStr(ctx, obj262, "detachArrayBuffer",
759                       JS_NewCFunction(ctx, js_detachArrayBuffer,
760                                       "detachArrayBuffer", 1));
761     JS_SetPropertyStr(ctx, obj262, "evalScript",
762                       JS_NewCFunction(ctx, js_evalScript,
763                                       "evalScript", 1));
764     JS_SetPropertyStr(ctx, obj262, "codePointRange",
765                       JS_NewCFunction(ctx, js_string_codePointRange,
766                                       "codePointRange", 2));
767 #ifdef CONFIG_AGENT
768     JS_SetPropertyStr(ctx, obj262, "agent", js_new_agent(ctx));
769 #endif
770 
771     JS_SetPropertyStr(ctx, obj262, "global",
772                       JS_DupValue(ctx, global_obj));
773     JS_SetPropertyStr(ctx, obj262, "createRealm",
774                       JS_NewCFunction(ctx, js_createRealm,
775                                       "createRealm", 0));
776 
777     JS_SetPropertyStr(ctx, global_obj, "$262", JS_DupValue(ctx, obj262));
778 
779     JS_FreeValue(ctx, global_obj);
780     return obj262;
781 }
782 
add_helpers(JSContext * ctx)783 static void add_helpers(JSContext *ctx)
784 {
785     JS_FreeValue(ctx, add_helpers1(ctx));
786 }
787 
load_file(const char * filename,size_t * lenp)788 static char *load_file(const char *filename, size_t *lenp)
789 {
790     char *buf;
791     size_t buf_len;
792     buf = (char *)js_load_file(NULL, &buf_len, filename);
793     if (!buf)
794         perror_exit(1, filename);
795     if (lenp)
796         *lenp = buf_len;
797     return buf;
798 }
799 
js_module_loader_test(JSContext * ctx,const char * module_name,void * opaque)800 static JSModuleDef *js_module_loader_test(JSContext *ctx,
801                                           const char *module_name, void *opaque)
802 {
803     size_t buf_len;
804     uint8_t *buf;
805     JSModuleDef *m;
806     JSValue func_val;
807 
808     buf = js_load_file(ctx, &buf_len, module_name);
809     if (!buf) {
810         JS_ThrowReferenceError(ctx, "could not load module filename '%s'",
811                                module_name);
812         return NULL;
813     }
814 
815     /* compile the module */
816     func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name,
817                        JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY);
818     js_free(ctx, buf);
819     if (JS_IsException(func_val))
820         return NULL;
821     /* the module is already referenced, so we must free it */
822     m = JS_VALUE_GET_PTR(func_val);
823     JS_FreeValue(ctx, func_val);
824     return m;
825 }
826 
is_line_sep(char c)827 int is_line_sep(char c)
828 {
829     return (c == '\0' || c == '\n' || c == '\r');
830 }
831 
find_line(const char * str,const char * line)832 char *find_line(const char *str, const char *line)
833 {
834     if (str) {
835         const char *p;
836         int len = strlen(line);
837         for (p = str; (p = strstr(p, line)) != NULL; p += len + 1) {
838             if ((p == str || is_line_sep(p[-1])) && is_line_sep(p[len]))
839                 return (char *)p;
840         }
841     }
842     return NULL;
843 }
844 
is_word_sep(char c)845 int is_word_sep(char c)
846 {
847     return (c == '\0' || isspace((unsigned char)c) || c == ',');
848 }
849 
find_word(const char * str,const char * word)850 char *find_word(const char *str, const char *word)
851 {
852     const char *p;
853     int len = strlen(word);
854     if (str && len) {
855         for (p = str; (p = strstr(p, word)) != NULL; p += len) {
856             if ((p == str || is_word_sep(p[-1])) && is_word_sep(p[len]))
857                 return (char *)p;
858         }
859     }
860     return NULL;
861 }
862 
863 /* handle exclude directories */
update_exclude_dirs(void)864 void update_exclude_dirs(void)
865 {
866     namelist_t *lp = &test_list;
867     namelist_t *ep = &exclude_list;
868     namelist_t *dp = &exclude_dir_list;
869     char *name;
870     int i, j, count;
871 
872     /* split directpries from exclude_list */
873     for (count = i = 0; i < ep->count; i++) {
874         name = ep->array[i];
875         if (has_suffix(name, "/")) {
876             namelist_add(dp, NULL, name);
877             free(name);
878         } else {
879             ep->array[count++] = name;
880         }
881     }
882     ep->count = count;
883 
884     namelist_sort(dp);
885 
886     /* filter out excluded directories */
887     for (count = i = 0; i < lp->count; i++) {
888         name = lp->array[i];
889         for (j = 0; j < dp->count; j++) {
890             if (has_prefix(name, dp->array[j])) {
891                 test_excluded++;
892                 free(name);
893                 name = NULL;
894                 break;
895             }
896         }
897         if (name) {
898             lp->array[count++] = name;
899         }
900     }
901     lp->count = count;
902 }
903 
load_config(const char * filename)904 void load_config(const char *filename)
905 {
906     char buf[1024];
907     FILE *f;
908     char *base_name;
909     enum {
910         SECTION_NONE = 0,
911         SECTION_CONFIG,
912         SECTION_EXCLUDE,
913         SECTION_FEATURES,
914         SECTION_TESTS,
915     } section = SECTION_NONE;
916     int lineno = 0;
917 
918     f = fopen(filename, "rb");
919     if (!f) {
920         perror_exit(1, filename);
921     }
922     base_name = get_basename(filename);
923 
924     while (fgets(buf, sizeof(buf), f) != NULL) {
925         char *p, *q;
926         lineno++;
927         p = str_strip(buf);
928         if (*p == '#' || *p == ';' || *p == '\0')
929             continue;  /* line comment */
930 
931         if (*p == "[]"[0]) {
932             /* new section */
933             p++;
934             p[strcspn(p, "]")] = '\0';
935             if (str_equal(p, "config"))
936                 section = SECTION_CONFIG;
937             else if (str_equal(p, "exclude"))
938                 section = SECTION_EXCLUDE;
939             else if (str_equal(p, "features"))
940                 section = SECTION_FEATURES;
941             else if (str_equal(p, "tests"))
942                 section = SECTION_TESTS;
943             else
944                 section = SECTION_NONE;
945             continue;
946         }
947         q = strchr(p, '=');
948         if (q) {
949             /* setting: name=value */
950             *q++ = '\0';
951             q = str_strip(q);
952         }
953         switch (section) {
954         case SECTION_CONFIG:
955             if (!q) {
956                 printf("%s:%d: syntax error\n", filename, lineno);
957                 continue;
958             }
959             if (str_equal(p, "style")) {
960                 new_style = str_equal(q, "new");
961                 continue;
962             }
963             if (str_equal(p, "testdir")) {
964                 char *testdir = compose_path(base_name, q);
965                 enumerate_tests(testdir);
966                 free(testdir);
967                 continue;
968             }
969             if (str_equal(p, "harnessdir")) {
970                 harness_dir = compose_path(base_name, q);
971                 continue;
972             }
973             if (str_equal(p, "harnessexclude")) {
974                 str_append(&harness_exclude, " ", q);
975                 continue;
976             }
977             if (str_equal(p, "features")) {
978                 str_append(&harness_features, " ", q);
979                 continue;
980             }
981             if (str_equal(p, "skip-features")) {
982                 str_append(&harness_skip_features, " ", q);
983                 continue;
984             }
985             if (str_equal(p, "mode")) {
986                 if (str_equal(q, "default") || str_equal(q, "default-nostrict"))
987                     test_mode = TEST_DEFAULT_NOSTRICT;
988                 else if (str_equal(q, "default-strict"))
989                     test_mode = TEST_DEFAULT_STRICT;
990                 else if (str_equal(q, "nostrict"))
991                     test_mode = TEST_NOSTRICT;
992                 else if (str_equal(q, "strict"))
993                     test_mode = TEST_STRICT;
994                 else if (str_equal(q, "all") || str_equal(q, "both"))
995                     test_mode = TEST_ALL;
996                 else
997                     fatal(2, "unknown test mode: %s", q);
998                 continue;
999             }
1000             if (str_equal(p, "strict")) {
1001                 if (str_equal(q, "skip") || str_equal(q, "no"))
1002                     test_mode = TEST_NOSTRICT;
1003                 continue;
1004             }
1005             if (str_equal(p, "nostrict")) {
1006                 if (str_equal(q, "skip") || str_equal(q, "no"))
1007                     test_mode = TEST_STRICT;
1008                 continue;
1009             }
1010             if (str_equal(p, "async")) {
1011                 skip_async = !str_equal(q, "yes");
1012                 continue;
1013             }
1014             if (str_equal(p, "module")) {
1015                 skip_module = !str_equal(q, "yes");
1016                 continue;
1017             }
1018             if (str_equal(p, "verbose")) {
1019                 verbose = str_equal(q, "yes");
1020                 continue;
1021             }
1022             if (str_equal(p, "errorfile")) {
1023                 error_filename = compose_path(base_name, q);
1024                 continue;
1025             }
1026             if (str_equal(p, "excludefile")) {
1027                 char *path = compose_path(base_name, q);
1028                 namelist_load(&exclude_list, path);
1029                 free(path);
1030                 continue;
1031             }
1032             if (str_equal(p, "reportfile")) {
1033                 report_filename = compose_path(base_name, q);
1034                 continue;
1035             }
1036         case SECTION_EXCLUDE:
1037             namelist_add(&exclude_list, base_name, p);
1038             break;
1039         case SECTION_FEATURES:
1040             if (!q || str_equal(q, "yes"))
1041                 str_append(&harness_features, " ", p);
1042             else
1043                 str_append(&harness_skip_features, " ", p);
1044             break;
1045         case SECTION_TESTS:
1046             namelist_add(&test_list, base_name, p);
1047             break;
1048         default:
1049             /* ignore settings in other sections */
1050             break;
1051         }
1052     }
1053     fclose(f);
1054     free(base_name);
1055 }
1056 
find_error(const char * filename,int * pline,int is_strict)1057 char *find_error(const char *filename, int *pline, int is_strict)
1058 {
1059     if (error_file) {
1060         size_t len = strlen(filename);
1061         const char *p, *q, *r;
1062         int line;
1063 
1064         for (p = error_file; (p = strstr(p, filename)) != NULL; p += len) {
1065             if ((p == error_file || p[-1] == '\n' || p[-1] == '(') && p[len] == ':') {
1066                 q = p + len;
1067                 line = 1;
1068                 if (*q == ':') {
1069                     line = strtol(q + 1, (char**)&q, 10);
1070                     if (*q == ':')
1071                         q++;
1072                 }
1073                 while (*q == ' ') {
1074                     q++;
1075                 }
1076                 /* check strict mode indicator */
1077                 if (!strstart(q, "strict mode: ", &q) != !is_strict)
1078                     continue;
1079                 r = q = skip_prefix(q, "unexpected error: ");
1080                 r += strcspn(r, "\n");
1081                 while (r[0] == '\n' && r[1] && strncmp(r + 1, filename, 8)) {
1082                     r++;
1083                     r += strcspn(r, "\n");
1084                 }
1085                 if (pline)
1086                     *pline = line;
1087                 return strdup_len(q, r - q);
1088             }
1089         }
1090     }
1091     return NULL;
1092 }
1093 
skip_comments(const char * str,int line,int * pline)1094 int skip_comments(const char *str, int line, int *pline)
1095 {
1096     const char *p;
1097     int c;
1098 
1099     p = str;
1100     while ((c = (unsigned char)*p++) != '\0') {
1101         if (isspace(c)) {
1102             if (c == '\n')
1103                 line++;
1104             continue;
1105         }
1106         if (c == '/' && *p == '/') {
1107             while (*++p && *p != '\n')
1108                 continue;
1109             continue;
1110         }
1111         if (c == '/' && *p == '*') {
1112             for (p += 1; *p; p++) {
1113                 if (*p == '\n') {
1114                     line++;
1115                     continue;
1116                 }
1117                 if (*p == '*' && p[1] == '/') {
1118                     p += 2;
1119                     break;
1120                 }
1121             }
1122             continue;
1123         }
1124         break;
1125     }
1126     if (pline)
1127         *pline = line;
1128 
1129     return p - str;
1130 }
1131 
longest_match(const char * str,const char * find,int pos,int * ppos,int line,int * pline)1132 int longest_match(const char *str, const char *find, int pos, int *ppos, int line, int *pline)
1133 {
1134     int len, maxlen;
1135 
1136     maxlen = 0;
1137 
1138     if (*find) {
1139         const char *p;
1140         for (p = str + pos; *p; p++) {
1141             if (*p == *find) {
1142                 for (len = 1; p[len] && p[len] == find[len]; len++)
1143                     continue;
1144                 if (len > maxlen) {
1145                     maxlen = len;
1146                     if (ppos)
1147                         *ppos = p - str;
1148                     if (pline)
1149                         *pline = line;
1150                     if (!find[len])
1151                         break;
1152                 }
1153             }
1154             if (*p == '\n')
1155                 line++;
1156         }
1157     }
1158     return maxlen;
1159 }
1160 
eval_buf(JSContext * ctx,const char * buf,size_t buf_len,const char * filename,int is_test,int is_negative,const char * error_type,FILE * outfile,int eval_flags,int is_async)1161 static int eval_buf(JSContext *ctx, const char *buf, size_t buf_len,
1162                     const char *filename, int is_test, int is_negative,
1163                     const char *error_type, FILE *outfile, int eval_flags,
1164                     int is_async)
1165 {
1166     JSValue res_val, exception_val;
1167     int ret, error_line, pos, pos_line;
1168     BOOL is_error, has_error_line;
1169     const char *error_name;
1170 
1171     pos = skip_comments(buf, 1, &pos_line);
1172     error_line = pos_line;
1173     has_error_line = FALSE;
1174     exception_val = JS_UNDEFINED;
1175     error_name = NULL;
1176 
1177     async_done = 0; /* counter of "Test262:AsyncTestComplete" messages */
1178 
1179     res_val = JS_Eval(ctx, buf, buf_len, filename, eval_flags);
1180 
1181     if (is_async && !JS_IsException(res_val)) {
1182         JS_FreeValue(ctx, res_val);
1183         for(;;) {
1184             JSContext *ctx1;
1185             ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
1186             if (ret < 0) {
1187                 res_val = JS_EXCEPTION;
1188                 break;
1189             } else if (ret == 0) {
1190                 /* test if the test called $DONE() once */
1191                 if (async_done != 1) {
1192                     res_val = JS_ThrowTypeError(ctx, "$DONE() not called");
1193                 } else {
1194                     res_val = JS_UNDEFINED;
1195                 }
1196                 break;
1197             }
1198         }
1199     }
1200 
1201     if (JS_IsException(res_val)) {
1202         exception_val = JS_GetException(ctx);
1203         is_error = JS_IsError(ctx, exception_val);
1204         /* XXX: should get the filename and line number */
1205         if (outfile) {
1206             if (!is_error)
1207                 fprintf(outfile, "%sThrow: ", (eval_flags & JS_EVAL_FLAG_STRICT) ?
1208                         "strict mode: " : "");
1209             js_print(ctx, JS_NULL, 1, &exception_val);
1210         }
1211         if (is_error) {
1212             JSValue name, stack;
1213             const char *stack_str;
1214 
1215             name = JS_GetPropertyStr(ctx, exception_val, "name");
1216             error_name = JS_ToCString(ctx, name);
1217             stack = JS_GetPropertyStr(ctx, exception_val, "stack");
1218             if (!JS_IsUndefined(stack)) {
1219                 stack_str = JS_ToCString(ctx, stack);
1220                 if (stack_str) {
1221                     const char *p;
1222                     int len;
1223 
1224                     if (outfile)
1225                         fprintf(outfile, "%s", stack_str);
1226 
1227                     len = strlen(filename);
1228                     p = strstr(stack_str, filename);
1229                     if (p != NULL && p[len] == ':') {
1230                         error_line = atoi(p + len + 1);
1231                         has_error_line = TRUE;
1232                     }
1233                     JS_FreeCString(ctx, stack_str);
1234                 }
1235             }
1236             JS_FreeValue(ctx, stack);
1237             JS_FreeValue(ctx, name);
1238         }
1239         if (is_negative) {
1240             ret = 0;
1241             if (error_type) {
1242                 char *error_class;
1243                 const char *msg;
1244 
1245                 msg = JS_ToCString(ctx, exception_val);
1246                 error_class = strdup_len(msg, strcspn(msg, ":"));
1247                 if (!str_equal(error_class, error_type))
1248                     ret = -1;
1249                 free(error_class);
1250                 JS_FreeCString(ctx, msg);
1251             }
1252         } else {
1253             ret = -1;
1254         }
1255     } else {
1256         if (is_negative)
1257             ret = -1;
1258         else
1259             ret = 0;
1260     }
1261 
1262     if (verbose && is_test) {
1263         JSValue msg_val = JS_UNDEFINED;
1264         const char *msg = NULL;
1265         int s_line;
1266         char *s = find_error(filename, &s_line, eval_flags & JS_EVAL_FLAG_STRICT);
1267         const char *strict_mode = (eval_flags & JS_EVAL_FLAG_STRICT) ? "strict mode: " : "";
1268 
1269         if (!JS_IsUndefined(exception_val)) {
1270             msg_val = JS_ToString(ctx, exception_val);
1271             msg = JS_ToCString(ctx, msg_val);
1272         }
1273         if (is_negative) {  // expect error
1274             if (ret == 0) {
1275                 if (msg && s &&
1276                     (str_equal(s, "expected error") ||
1277                      strstart(s, "unexpected error type:", NULL) ||
1278                      str_equal(s, msg))) {     // did not have error yet
1279                     if (!has_error_line) {
1280                         longest_match(buf, msg, pos, &pos, pos_line, &error_line);
1281                     }
1282                     printf("%s:%d: %sOK, now has error %s\n",
1283                            filename, error_line, strict_mode, msg);
1284                     fixed_errors++;
1285                 }
1286             } else {
1287                 if (!s) {   // not yet reported
1288                     if (msg) {
1289                         fprintf(error_out, "%s:%d: %sunexpected error type: %s\n",
1290                                 filename, error_line, strict_mode, msg);
1291                     } else {
1292                         fprintf(error_out, "%s:%d: %sexpected error\n",
1293                                 filename, error_line, strict_mode);
1294                     }
1295                     new_errors++;
1296                 }
1297             }
1298         } else {            // should not have error
1299             if (msg) {
1300                 if (!s || !str_equal(s, msg)) {
1301                     if (!has_error_line) {
1302                         char *p = skip_prefix(msg, "Test262 Error: ");
1303                         if (strstr(p, "Test case returned non-true value!")) {
1304                             longest_match(buf, "runTestCase", pos, &pos, pos_line, &error_line);
1305                         } else {
1306                             longest_match(buf, p, pos, &pos, pos_line, &error_line);
1307                         }
1308                     }
1309                     fprintf(error_out, "%s:%d: %s%s%s\n", filename, error_line, strict_mode,
1310                             error_file ? "unexpected error: " : "", msg);
1311 
1312                     if (s && (!str_equal(s, msg) || error_line != s_line)) {
1313                         printf("%s:%d: %sprevious error: %s\n", filename, s_line, strict_mode, s);
1314                         changed_errors++;
1315                     } else {
1316                         new_errors++;
1317                     }
1318                 }
1319             } else {
1320                 if (s) {
1321                     printf("%s:%d: %sOK, fixed error: %s\n", filename, s_line, strict_mode, s);
1322                     fixed_errors++;
1323                 }
1324             }
1325         }
1326         JS_FreeValue(ctx, msg_val);
1327         JS_FreeCString(ctx, msg);
1328         free(s);
1329     }
1330     JS_FreeCString(ctx, error_name);
1331     JS_FreeValue(ctx, exception_val);
1332     JS_FreeValue(ctx, res_val);
1333     return ret;
1334 }
1335 
eval_file(JSContext * ctx,const char * base,const char * p,int eval_flags)1336 static int eval_file(JSContext *ctx, const char *base, const char *p,
1337                      int eval_flags)
1338 {
1339     char *buf;
1340     size_t buf_len;
1341     char *filename = compose_path(base, p);
1342 
1343     buf = load_file(filename, &buf_len);
1344     if (!buf) {
1345         warning("cannot load %s", filename);
1346         goto fail;
1347     }
1348     if (eval_buf(ctx, buf, buf_len, filename, FALSE, FALSE, NULL, stderr,
1349                  eval_flags, FALSE)) {
1350         warning("error evaluating %s", filename);
1351         goto fail;
1352     }
1353     free(buf);
1354     free(filename);
1355     return 0;
1356 
1357 fail:
1358     free(buf);
1359     free(filename);
1360     return 1;
1361 }
1362 
extract_desc(const char * buf,char style)1363 char *extract_desc(const char *buf, char style)
1364 {
1365     const char *p, *desc_start;
1366     char *desc;
1367     int len;
1368 
1369     p = buf;
1370     while (*p != '\0') {
1371         if (p[0] == '/' && p[1] == '*' && p[2] == style && p[3] != '/') {
1372             p += 3;
1373             desc_start = p;
1374             while (*p != '\0' && (p[0] != '*' || p[1] != '/'))
1375                 p++;
1376             if (*p == '\0') {
1377                 warning("Expecting end of desc comment");
1378                 return NULL;
1379             }
1380             len = p - desc_start;
1381             desc = malloc(len + 1);
1382             memcpy(desc, desc_start, len);
1383             desc[len] = '\0';
1384             return desc;
1385         } else {
1386             p++;
1387         }
1388     }
1389     return NULL;
1390 }
1391 
find_tag(char * desc,const char * tag,int * state)1392 static char *find_tag(char *desc, const char *tag, int *state)
1393 {
1394     char *p;
1395     p = strstr(desc, tag);
1396     if (p) {
1397         p += strlen(tag);
1398         *state = 0;
1399     }
1400     return p;
1401 }
1402 
get_option(char ** pp,int * state)1403 static char *get_option(char **pp, int *state)
1404 {
1405     char *p, *p0, *option = NULL;
1406     if (*pp) {
1407         for (p = *pp;; p++) {
1408             switch (*p) {
1409             case '[':
1410                 *state += 1;
1411                 continue;
1412             case ']':
1413                 *state -= 1;
1414                 if (*state > 0)
1415                     continue;
1416                 p = NULL;
1417                 break;
1418             case ' ':
1419             case '\t':
1420             case '\r':
1421             case ',':
1422             case '-':
1423                 continue;
1424             case '\n':
1425                 if (*state > 0 || p[1] == ' ')
1426                     continue;
1427                 p = NULL;
1428                 break;
1429             case '\0':
1430                 p = NULL;
1431                 break;
1432             default:
1433                 p0 = p;
1434                 p += strcspn(p0, " \t\r\n,]");
1435                 option = strdup_len(p0, p - p0);
1436                 break;
1437             }
1438             break;
1439         }
1440         *pp = p;
1441     }
1442     return option;
1443 }
1444 
update_stats(JSRuntime * rt,const char * filename)1445 void update_stats(JSRuntime *rt, const char *filename) {
1446     JSMemoryUsage stats;
1447     JS_ComputeMemoryUsage(rt, &stats);
1448     if (stats_count++ == 0) {
1449         stats_avg = stats_all = stats_min = stats_max = stats;
1450         stats_min_filename = strdup(filename);
1451         stats_max_filename = strdup(filename);
1452     } else {
1453         if (stats_max.malloc_size < stats.malloc_size) {
1454             stats_max = stats;
1455             free(stats_max_filename);
1456             stats_max_filename = strdup(filename);
1457         }
1458         if (stats_min.malloc_size > stats.malloc_size) {
1459             stats_min = stats;
1460             free(stats_min_filename);
1461             stats_min_filename = strdup(filename);
1462         }
1463 
1464 #define update(f)  stats_avg.f = (stats_all.f += stats.f) / stats_count
1465         update(malloc_count);
1466         update(malloc_size);
1467         update(memory_used_count);
1468         update(memory_used_size);
1469         update(atom_count);
1470         update(atom_size);
1471         update(str_count);
1472         update(str_size);
1473         update(obj_count);
1474         update(obj_size);
1475         update(prop_count);
1476         update(prop_size);
1477         update(shape_count);
1478         update(shape_size);
1479         update(js_func_count);
1480         update(js_func_size);
1481         update(js_func_code_size);
1482         update(js_func_pc2line_count);
1483         update(js_func_pc2line_size);
1484         update(c_func_count);
1485         update(array_count);
1486         update(fast_array_count);
1487         update(fast_array_elements);
1488     }
1489 #undef update
1490 }
1491 
run_test_buf(const char * filename,char * harness,namelist_t * ip,char * buf,size_t buf_len,const char * error_type,int eval_flags,BOOL is_negative,BOOL is_async,BOOL can_block)1492 int run_test_buf(const char *filename, char *harness, namelist_t *ip,
1493                  char *buf, size_t buf_len, const char* error_type,
1494                  int eval_flags, BOOL is_negative, BOOL is_async,
1495                  BOOL can_block)
1496 {
1497     JSRuntime *rt;
1498     JSContext *ctx;
1499     int i, ret;
1500 
1501     rt = JS_NewRuntime();
1502     if (rt == NULL) {
1503         fatal(1, "JS_NewRuntime failure");
1504     }
1505     ctx = JS_NewContext(rt);
1506     if (ctx == NULL) {
1507         JS_FreeRuntime(rt);
1508         fatal(1, "JS_NewContext failure");
1509     }
1510     JS_SetRuntimeInfo(rt, filename);
1511 
1512     JS_SetCanBlock(rt, can_block);
1513 
1514     /* loader for ES6 modules */
1515     JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, NULL);
1516 
1517     add_helpers(ctx);
1518 
1519     for (i = 0; i < ip->count; i++) {
1520         if (eval_file(ctx, harness, ip->array[i],
1521                       JS_EVAL_TYPE_GLOBAL | JS_EVAL_FLAG_STRIP)) {
1522             fatal(1, "error including %s for %s", ip->array[i], filename);
1523         }
1524     }
1525 
1526     ret = eval_buf(ctx, buf, buf_len, filename, TRUE, is_negative,
1527                    error_type, outfile, eval_flags, is_async);
1528     ret = (ret != 0);
1529 
1530     if (dump_memory) {
1531         update_stats(rt, filename);
1532     }
1533 #ifdef CONFIG_AGENT
1534     js_agent_free(ctx);
1535 #endif
1536     JS_FreeContext(ctx);
1537     JS_FreeRuntime(rt);
1538 
1539     test_count++;
1540     if (ret) {
1541         test_failed++;
1542         if (outfile) {
1543             /* do not output a failure number to minimize diff */
1544             fprintf(outfile, "  FAILED\n");
1545         }
1546     }
1547     return ret;
1548 }
1549 
run_test(const char * filename,int index)1550 int run_test(const char *filename, int index)
1551 {
1552     char harnessbuf[1024];
1553     char *harness;
1554     char *buf;
1555     size_t buf_len;
1556     char *desc, *p;
1557     char *error_type;
1558     int ret, eval_flags, use_strict, use_nostrict;
1559     BOOL is_negative, is_nostrict, is_onlystrict, is_async, is_module, skip;
1560     BOOL can_block;
1561     namelist_t include_list = { 0 }, *ip = &include_list;
1562 
1563     is_nostrict = is_onlystrict = is_negative = is_async = is_module = skip = FALSE;
1564     can_block = TRUE;
1565     error_type = NULL;
1566     buf = load_file(filename, &buf_len);
1567 
1568     harness = harness_dir;
1569 
1570     if (new_style) {
1571         if (!harness) {
1572             p = strstr(filename, "test/");
1573             if (p) {
1574                 snprintf(harnessbuf, sizeof(harnessbuf), "%.*s%s",
1575                          (int)(p - filename), filename, "harness");
1576             }
1577             harness = harnessbuf;
1578         }
1579         namelist_add(ip, NULL, "sta.js");
1580         namelist_add(ip, NULL, "assert.js");
1581         /* extract the YAML frontmatter */
1582         desc = extract_desc(buf, '-');
1583         if (desc) {
1584             char *ifile, *option;
1585             int state;
1586             p = find_tag(desc, "includes:", &state);
1587             if (p) {
1588                 while ((ifile = get_option(&p, &state)) != NULL) {
1589                     // skip unsupported harness files
1590                     if (find_word(harness_exclude, ifile)) {
1591                         skip |= 1;
1592                     } else {
1593                         namelist_add(ip, NULL, ifile);
1594                     }
1595                     free(ifile);
1596                 }
1597             }
1598             p = find_tag(desc, "flags:", &state);
1599             if (p) {
1600                 while ((option = get_option(&p, &state)) != NULL) {
1601                     if (str_equal(option, "noStrict") ||
1602                         str_equal(option, "raw")) {
1603                         is_nostrict = TRUE;
1604                         skip |= (test_mode == TEST_STRICT);
1605                     }
1606                     else if (str_equal(option, "onlyStrict")) {
1607                         is_onlystrict = TRUE;
1608                         skip |= (test_mode == TEST_NOSTRICT);
1609                     }
1610                     else if (str_equal(option, "async")) {
1611                         is_async = TRUE;
1612                         skip |= skip_async;
1613                     }
1614                     else if (str_equal(option, "module")) {
1615                         is_module = TRUE;
1616                         skip |= skip_module;
1617                     }
1618                     else if (str_equal(option, "CanBlockIsFalse")) {
1619                         can_block = FALSE;
1620                     }
1621                     free(option);
1622                 }
1623             }
1624             p = find_tag(desc, "negative:", &state);
1625             if (p) {
1626                 /* XXX: should extract the phase */
1627                 char *q = find_tag(p, "type:", &state);
1628                 if (q) {
1629                     while (isspace(*q))
1630                         q++;
1631                     error_type = strdup_len(q, strcspn(q, " \n"));
1632                 }
1633                 is_negative = TRUE;
1634             }
1635             p = find_tag(desc, "features:", &state);
1636             if (p) {
1637                 while ((option = get_option(&p, &state)) != NULL) {
1638                     if (find_word(harness_features, option)) {
1639                         /* feature is enabled */
1640                     } else if (find_word(harness_skip_features, option)) {
1641                         /* skip disabled feature */
1642                         skip |= 1;
1643                     } else {
1644                         /* feature is not listed: skip and warn */
1645                         printf("%s:%d: unknown feature: %s\n", filename, 1, option);
1646                         skip |= 1;
1647                     }
1648                     free(option);
1649                 }
1650             }
1651             free(desc);
1652         }
1653         if (is_async)
1654             namelist_add(ip, NULL, "doneprintHandle.js");
1655     } else {
1656         char *ifile;
1657 
1658         if (!harness) {
1659             p = strstr(filename, "test/");
1660             if (p) {
1661                 snprintf(harnessbuf, sizeof(harnessbuf), "%.*s%s",
1662                          (int)(p - filename), filename, "test/harness");
1663             }
1664             harness = harnessbuf;
1665         }
1666 
1667         namelist_add(ip, NULL, "sta.js");
1668 
1669         /* include extra harness files */
1670         for (p = buf; (p = strstr(p, "$INCLUDE(\"")) != NULL; p++) {
1671             p += 10;
1672             ifile = strdup_len(p, strcspn(p, "\""));
1673             // skip unsupported harness files
1674             if (find_word(harness_exclude, ifile)) {
1675                 skip |= 1;
1676             } else {
1677                 namelist_add(ip, NULL, ifile);
1678             }
1679             free(ifile);
1680         }
1681 
1682         /* locate the old style configuration comment */
1683         desc = extract_desc(buf, '*');
1684         if (desc) {
1685             if (strstr(desc, "@noStrict")) {
1686                 is_nostrict = TRUE;
1687                 skip |= (test_mode == TEST_STRICT);
1688             }
1689             if (strstr(desc, "@onlyStrict")) {
1690                 is_onlystrict = TRUE;
1691                 skip |= (test_mode == TEST_NOSTRICT);
1692             }
1693             if (strstr(desc, "@negative")) {
1694                 /* XXX: should extract the regex to check error type */
1695                 is_negative = TRUE;
1696             }
1697             free(desc);
1698         }
1699     }
1700 
1701     if (outfile && index >= 0) {
1702         fprintf(outfile, "%d: %s%s%s%s%s%s%s\n", index, filename,
1703                 is_nostrict ? "  @noStrict" : "",
1704                 is_onlystrict ? "  @onlyStrict" : "",
1705                 is_async ? "  async" : "",
1706                 is_module ? "  module" : "",
1707                 is_negative ? "  @negative" : "",
1708                 skip ? "  SKIPPED" : "");
1709         fflush(outfile);
1710     }
1711 
1712     use_strict = use_nostrict = 0;
1713     /* XXX: should remove 'test_mode' or simplify it just to force
1714        strict or non strict mode for single file tests */
1715     switch (test_mode) {
1716     case TEST_DEFAULT_NOSTRICT:
1717         if (is_onlystrict)
1718             use_strict = 1;
1719         else
1720             use_nostrict = 1;
1721         break;
1722     case TEST_DEFAULT_STRICT:
1723         if (is_nostrict)
1724             use_nostrict = 1;
1725         else
1726             use_strict = 1;
1727         break;
1728     case TEST_NOSTRICT:
1729         if (!is_onlystrict)
1730             use_nostrict = 1;
1731         break;
1732     case TEST_STRICT:
1733         if (!is_nostrict)
1734             use_strict = 1;
1735         break;
1736     case TEST_ALL:
1737         if (is_module) {
1738             use_nostrict = 1;
1739         } else {
1740             if (!is_nostrict)
1741                 use_strict = 1;
1742             if (!is_onlystrict)
1743                 use_nostrict = 1;
1744         }
1745         break;
1746     }
1747 
1748     if (skip || use_strict + use_nostrict == 0) {
1749         test_skipped++;
1750         ret = -2;
1751     } else {
1752         clock_t clocks;
1753 
1754         if (is_module) {
1755             eval_flags = JS_EVAL_TYPE_MODULE;
1756         } else {
1757             eval_flags = JS_EVAL_TYPE_GLOBAL;
1758         }
1759         clocks = clock();
1760         ret = 0;
1761         if (use_nostrict) {
1762             ret = run_test_buf(filename, harness, ip, buf, buf_len,
1763                                error_type, eval_flags, is_negative, is_async,
1764                                can_block);
1765         }
1766         if (use_strict) {
1767             ret |= run_test_buf(filename, harness, ip, buf, buf_len,
1768                                 error_type, eval_flags | JS_EVAL_FLAG_STRICT,
1769                                 is_negative, is_async, can_block);
1770         }
1771         clocks = clock() - clocks;
1772         if (outfile && index >= 0 && clocks >= CLOCKS_PER_SEC / 10) {
1773             /* output timings for tests that take more than 100 ms */
1774             fprintf(outfile, " time: %d ms\n", (int)(clocks * 1000LL / CLOCKS_PER_SEC));
1775         }
1776     }
1777     namelist_free(&include_list);
1778     free(error_type);
1779     free(buf);
1780 
1781     return ret;
1782 }
1783 
1784 /* run a test when called by test262-harness+eshost */
run_test262_harness_test(const char * filename,BOOL is_module)1785 int run_test262_harness_test(const char *filename, BOOL is_module)
1786 {
1787     JSRuntime *rt;
1788     JSContext *ctx;
1789     char *buf;
1790     size_t buf_len;
1791     int eval_flags, ret_code, ret;
1792     JSValue res_val;
1793     BOOL can_block;
1794 
1795     outfile = stdout; /* for js_print */
1796 
1797     rt = JS_NewRuntime();
1798     if (rt == NULL) {
1799         fatal(1, "JS_NewRuntime failure");
1800     }
1801     ctx = JS_NewContext(rt);
1802     if (ctx == NULL) {
1803         JS_FreeRuntime(rt);
1804         fatal(1, "JS_NewContext failure");
1805     }
1806     JS_SetRuntimeInfo(rt, filename);
1807 
1808     can_block = TRUE;
1809     JS_SetCanBlock(rt, can_block);
1810 
1811     /* loader for ES6 modules */
1812     JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, NULL);
1813 
1814     add_helpers(ctx);
1815 
1816     buf = load_file(filename, &buf_len);
1817 
1818     if (is_module) {
1819       eval_flags = JS_EVAL_TYPE_MODULE;
1820     } else {
1821       eval_flags = JS_EVAL_TYPE_GLOBAL;
1822     }
1823     res_val = JS_Eval(ctx, buf, buf_len, filename, eval_flags);
1824     ret_code = 0;
1825     if (JS_IsException(res_val)) {
1826        js_std_dump_error(ctx);
1827        ret_code = 1;
1828     } else {
1829         JS_FreeValue(ctx, res_val);
1830         for(;;) {
1831             JSContext *ctx1;
1832             ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
1833             if (ret < 0) {
1834 	      js_std_dump_error(ctx1);
1835 	      ret_code = 1;
1836             } else if (ret == 0) {
1837 	      break;
1838             }
1839         }
1840     }
1841     free(buf);
1842 #ifdef CONFIG_AGENT
1843     js_agent_free(ctx);
1844 #endif
1845     JS_FreeContext(ctx);
1846     JS_FreeRuntime(rt);
1847     return ret_code;
1848 }
1849 
1850 clock_t last_clock;
1851 
show_progress(int force)1852 void show_progress(int force) {
1853     clock_t t = clock();
1854     if (force || !last_clock || (t - last_clock) > CLOCKS_PER_SEC / 20) {
1855         last_clock = t;
1856         /* output progress indicator: erase end of line and return to col 0 */
1857         fprintf(stderr, "%d/%d/%d\033[K\r",
1858                 test_failed, test_count, test_skipped);
1859         fflush(stderr);
1860     }
1861 }
1862 
1863 static int slow_test_threshold;
1864 
run_test_dir_list(namelist_t * lp,int start_index,int stop_index)1865 void run_test_dir_list(namelist_t *lp, int start_index, int stop_index)
1866 {
1867     int i;
1868 
1869     namelist_sort(lp);
1870     for (i = 0; i < lp->count; i++) {
1871         const char *p = lp->array[i];
1872         if (namelist_find(&exclude_list, p) >= 0) {
1873             test_excluded++;
1874         } else if (test_index < start_index) {
1875             test_skipped++;
1876         } else if (stop_index >= 0 && test_index > stop_index) {
1877             test_skipped++;
1878         } else {
1879             int ti;
1880             if (slow_test_threshold != 0) {
1881                 ti = get_clock_ms();
1882             } else {
1883                 ti = 0;
1884             }
1885             run_test(p, test_index);
1886             if (slow_test_threshold != 0) {
1887                 ti = get_clock_ms() - ti;
1888                 if (ti >= slow_test_threshold)
1889                     fprintf(stderr, "\n%s (%d ms)\n", p, ti);
1890             }
1891             show_progress(FALSE);
1892         }
1893         test_index++;
1894     }
1895     show_progress(TRUE);
1896 }
1897 
help(void)1898 void help(void)
1899 {
1900     printf("run-test262 version " CONFIG_VERSION "\n"
1901            "usage: run-test262 [options] {-f file ... | [dir_list] [index range]}\n"
1902            "-h             help\n"
1903            "-a             run tests in strict and nostrict modes\n"
1904            "-m             print memory usage summary\n"
1905            "-n             use new style harness\n"
1906            "-N             run test prepared by test262-harness+eshost\n"
1907            "-s             run tests in strict mode, skip @nostrict tests\n"
1908            "-E             only run tests from the error file\n"
1909            "-u             update error file\n"
1910            "-v             verbose: output error messages\n"
1911            "-T duration    display tests taking more than 'duration' ms\n"
1912            "-c file        read configuration from 'file'\n"
1913            "-d dir         run all test files in directory tree 'dir'\n"
1914            "-e file        load the known errors from 'file'\n"
1915            "-f file        execute single test from 'file'\n"
1916            "-r file        set the report file name (default=none)\n"
1917            "-x file        exclude tests listed in 'file'\n");
1918     exit(1);
1919 }
1920 
get_opt_arg(const char * option,char * arg)1921 char *get_opt_arg(const char *option, char *arg)
1922 {
1923     if (!arg) {
1924         fatal(2, "missing argument for option %s", option);
1925     }
1926     return arg;
1927 }
1928 
main(int argc,char ** argv)1929 int main(int argc, char **argv)
1930 {
1931     int optind, start_index, stop_index;
1932     BOOL is_dir_list;
1933     BOOL only_check_errors = FALSE;
1934     const char *filename;
1935     BOOL is_test262_harness = FALSE;
1936     BOOL is_module = FALSE;
1937 
1938 #if !defined(_WIN32)
1939     /* Date tests assume California local time */
1940     setenv("TZ", "America/Los_Angeles", 1);
1941 #endif
1942 
1943     /* cannot use getopt because we want to pass the command line to
1944        the script */
1945     optind = 1;
1946     is_dir_list = TRUE;
1947     while (optind < argc) {
1948         char *arg = argv[optind];
1949         if (*arg != '-')
1950             break;
1951         optind++;
1952         if (str_equal(arg, "-h")) {
1953             help();
1954         } else if (str_equal(arg, "-m")) {
1955             dump_memory++;
1956         } else if (str_equal(arg, "-n")) {
1957             new_style++;
1958         } else if (str_equal(arg, "-s")) {
1959             test_mode = TEST_STRICT;
1960         } else if (str_equal(arg, "-a")) {
1961             test_mode = TEST_ALL;
1962         } else if (str_equal(arg, "-u")) {
1963             update_errors++;
1964         } else if (str_equal(arg, "-v")) {
1965             verbose++;
1966         } else if (str_equal(arg, "-c")) {
1967             load_config(get_opt_arg(arg, argv[optind++]));
1968         } else if (str_equal(arg, "-d")) {
1969             enumerate_tests(get_opt_arg(arg, argv[optind++]));
1970         } else if (str_equal(arg, "-e")) {
1971             error_filename = get_opt_arg(arg, argv[optind++]);
1972         } else if (str_equal(arg, "-x")) {
1973             namelist_load(&exclude_list, get_opt_arg(arg, argv[optind++]));
1974         } else if (str_equal(arg, "-f")) {
1975             is_dir_list = FALSE;
1976         } else if (str_equal(arg, "-r")) {
1977             report_filename = get_opt_arg(arg, argv[optind++]);
1978         } else if (str_equal(arg, "-E")) {
1979             only_check_errors = TRUE;
1980         } else if (str_equal(arg, "-T")) {
1981             slow_test_threshold = atoi(get_opt_arg(arg, argv[optind++]));
1982         } else if (str_equal(arg, "-N")) {
1983             is_test262_harness = TRUE;
1984         } else if (str_equal(arg, "--module")) {
1985             is_module = TRUE;
1986         } else {
1987             fatal(1, "unknown option: %s", arg);
1988             break;
1989         }
1990     }
1991 
1992     if (optind >= argc && !test_list.count)
1993         help();
1994 
1995     if (is_test262_harness) {
1996         return run_test262_harness_test(argv[optind], is_module);
1997     }
1998 
1999     error_out = stdout;
2000     if (error_filename) {
2001         error_file = load_file(error_filename, NULL);
2002         if (only_check_errors && error_file) {
2003             namelist_free(&test_list);
2004             namelist_add_from_error_file(&test_list, error_file);
2005         }
2006         if (update_errors) {
2007             free(error_file);
2008             error_file = NULL;
2009             error_out = fopen(error_filename, "w");
2010             if (!error_out) {
2011                 perror_exit(1, error_filename);
2012             }
2013         }
2014     }
2015 
2016     update_exclude_dirs();
2017 
2018     if (is_dir_list) {
2019         if (optind < argc && !isdigit(argv[optind][0])) {
2020             filename = argv[optind++];
2021             namelist_load(&test_list, filename);
2022         }
2023         start_index = 0;
2024         stop_index = -1;
2025         if (optind < argc) {
2026             start_index = atoi(argv[optind++]);
2027             if (optind < argc) {
2028                 stop_index = atoi(argv[optind++]);
2029             }
2030         }
2031         if (!report_filename || str_equal(report_filename, "none")) {
2032             outfile = NULL;
2033         } else if (str_equal(report_filename, "-")) {
2034             outfile = stdout;
2035         } else {
2036             outfile = fopen(report_filename, "wb");
2037             if (!outfile) {
2038                 perror_exit(1, report_filename);
2039             }
2040         }
2041         run_test_dir_list(&test_list, start_index, stop_index);
2042 
2043         if (outfile && outfile != stdout) {
2044             fclose(outfile);
2045             outfile = NULL;
2046         }
2047     } else {
2048         outfile = stdout;
2049         while (optind < argc) {
2050             run_test(argv[optind++], -1);
2051         }
2052     }
2053 
2054     if (dump_memory) {
2055         if (dump_memory > 1 && stats_count > 1) {
2056             printf("\nMininum memory statistics for %s:\n\n", stats_min_filename);
2057             JS_DumpMemoryUsage(stdout, &stats_min, NULL);
2058             printf("\nMaximum memory statistics for %s:\n\n", stats_max_filename);
2059             JS_DumpMemoryUsage(stdout, &stats_max, NULL);
2060         }
2061         printf("\nAverage memory statistics for %d tests:\n\n", stats_count);
2062         JS_DumpMemoryUsage(stdout, &stats_avg, NULL);
2063         printf("\n");
2064     }
2065 
2066     if (is_dir_list) {
2067         fprintf(stderr, "Result: %d/%d error%s",
2068                 test_failed, test_count, test_count != 1 ? "s" : "");
2069         if (test_excluded)
2070             fprintf(stderr, ", %d excluded", test_excluded);
2071         if (test_skipped)
2072             fprintf(stderr, ", %d skipped", test_skipped);
2073         if (error_file) {
2074             if (new_errors)
2075                 fprintf(stderr, ", %d new", new_errors);
2076             if (changed_errors)
2077                 fprintf(stderr, ", %d changed", changed_errors);
2078             if (fixed_errors)
2079                 fprintf(stderr, ", %d fixed", fixed_errors);
2080         }
2081         fprintf(stderr, "\n");
2082     }
2083 
2084     if (error_out && error_out != stdout) {
2085         fclose(error_out);
2086         error_out = NULL;
2087     }
2088 
2089     namelist_free(&test_list);
2090     namelist_free(&exclude_list);
2091     namelist_free(&exclude_dir_list);
2092     free(harness_dir);
2093     free(harness_features);
2094     free(harness_exclude);
2095     free(error_file);
2096 
2097     return 0;
2098 }
2099