1#!/usr/bin/env bash
2set -e
3
4TOOLCHAINS_CSV='support/config-fragments/autobuild/toolchain-configs.csv'
5TEMP_CONF=""
6
7do_clean() {
8    if [ -n "${TEMP_CONF}" ]; then
9        rm -f "${TEMP_CONF}"
10    fi
11}
12
13main() {
14    local o O opts
15    local cfg dir pkg random toolchains_csv toolchain all number mode prepare_only
16    local ret nb nb_skip nb_fail nb_legal nb_show nb_tc build_dir keep
17    local -a toolchains
18    local pkg_br_name
19
20    o='hakc:d:n:p:r:t:'
21    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:'
22    opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
23    eval set -- "${opts}"
24
25    random=0
26    all=0
27    keep=0
28    number=0
29    mode=0
30    prepare_only=0
31    toolchains_csv="${TOOLCHAINS_CSV}"
32    while [ ${#} -gt 0 ]; do
33        case "${1}" in
34        (-h|--help)
35            help; exit 0
36            ;;
37        (-a|--all)
38            all=1; shift 1
39            ;;
40        (-k|--keep)
41            keep=1; shift 1
42            ;;
43        (--prepare-only)
44            prepare_only=1; shift 1
45            ;;
46        (-c|--config-snippet)
47            cfg="${2}"; shift 2
48            ;;
49        (-d|--build-dir)
50            dir="${2}"; shift 2
51            ;;
52        (-n|--number)
53            number="${2}"; shift 2
54            ;;
55        (-p|--package)
56            pkg="${2}"; shift 2
57            ;;
58        (-r|--random)
59            random="${2}"; shift 2
60            ;;
61        (-t|--toolchains-csv)
62            toolchains_csv="${2}"; shift 2
63            ;;
64        (--)
65            shift; break
66            ;;
67        esac
68    done
69
70    trap do_clean INT TERM HUP EXIT
71
72    if [ -z "${cfg}" ]; then
73        pkg_br_name="${pkg//-/_}"
74        pkg_br_name="BR2_PACKAGE_${pkg_br_name^^}"
75        TEMP_CONF="$(mktemp /tmp/test-"${pkg}"-config.XXXXXX)"
76        echo "${pkg_br_name}=y" > "${TEMP_CONF}"
77        cfg="${TEMP_CONF}"
78    fi
79    if [ ! -e "${cfg}" ]; then
80        printf "error: %s: no such file\n" "${cfg}" >&2; exit 1
81    fi
82    if [ -z "${dir}" ]; then
83        dir="${HOME}/br-test-pkg"
84    fi
85
86    if [ "${random}" -gt 0 ]; then
87        mode=$((mode+1))
88    fi
89
90    if [ "${number}" -gt 0 ]; then
91        mode=$((mode+1))
92    fi
93
94    if [ "${all}" -eq 1 ]; then
95        mode=$((mode+1))
96    fi
97
98    # Default mode is to test the N first toolchains, which have been
99    # chosen to be a good selection of toolchains.
100    if [ ${mode} -eq 0 ] ; then
101        number=6
102    elif [ ${mode} -gt 1 ] ; then
103        printf "error: --all, --number and --random are mutually exclusive\n" >&2; exit 1
104    fi
105
106    # Extract the URLs of the toolchains; drop internal toolchains
107    # E.g.: http://server/path/to/name.config,arch,libc
108    #  -->  http://server/path/to/name.config
109    mapfile -t toolchains < <(sed -r -e 's/,.*//; /internal/d; /^#/d; /^$/d;' "${toolchains_csv}" \
110                                | if [ "${random}" -gt 0 ]; then \
111                                    sort -R | head -n "${random}"
112                                elif [ "${number}" -gt 0 ]; then \
113                                    head -n "${number}"
114                                else
115                                    sort
116                                fi
117                             )
118
119    nb_tc="${#toolchains[@]}"
120    if [ "${nb_tc}" -eq 0 ]; then
121        printf "error: no toolchain found (networking issue?)\n" >&2; exit 1
122    fi
123
124    nb=0
125    nb_skip=0
126    nb_fail=0
127    nb_legal=0
128    nb_show=0
129    for toolchainconfig in "${toolchains[@]}"; do
130        : $((nb++))
131        toolchain="$(basename "${toolchainconfig}" .config)"
132        build_dir="${dir}/${toolchain}"
133        printf "%40s [%*d/%d]: " "${toolchain}" ${#nb_tc} "${nb}" "${nb_tc}"
134        build_one "${build_dir}" "${toolchainconfig}" "${cfg}" "${pkg}" "${prepare_only}" && ret=0 || ret=${?}
135        case ${ret} in
136        (0) printf "OK\n";;
137        (1) : $((nb_skip++)); printf "SKIPPED\n";;
138        (2) : $((nb_fail++)); printf "FAILED\n";;
139        (3) : $((nb_legal++)); printf "FAILED\n";;
140        (4) : $((nb_show++)); printf "FAILED\n";;
141        esac
142    done
143
144    printf "%d builds, %d skipped, %d build failed, %d legal-info failed, %d show-info failed\n" \
145        "${nb}" "${nb_skip}" "${nb_fail}" "${nb_legal}" "${nb_show}"
146
147    return $((nb_fail + nb_legal))
148}
149
150build_one() {
151    local dir="${1}"
152    local toolchainconfig="${2}"
153    local cfg="${3}"
154    local pkg="${4}"
155    local prepare_only="${5}"
156
157    mkdir -p "${dir}"
158
159    CONFIG_="" support/kconfig/merge_config.sh -O "${dir}" \
160        "${toolchainconfig}" "support/config-fragments/minimal.config" "${cfg}" \
161        >> "${dir}/logfile" 2>&1
162    # We want all the options from the snippet to be present as-is (set
163    # or not set) in the actual .config; if one of them is not, it means
164    # some dependency from the toolchain or arch is not available, in
165    # which case this config is untestable and we skip it.
166    # We don't care about the locale to sort in, as long as both sort are
167    # done in the same locale.
168    comm -23 <(sort "${cfg}") <(sort "${dir}/.config") >"${dir}/missing.config"
169    if [ -s "${dir}/missing.config" ]; then
170        if [ ${keep} -ne 1 ]; then
171            # Invalid configuration, drop it
172            rm -f "${dir}/.config"
173        fi
174        return 1
175    fi
176    # Remove file, it's empty anyway.
177    rm -f "${dir}/missing.config"
178
179    # Defer building the job to the caller (e.g. a gitlab pipeline)
180    if [ "${prepare_only}" -eq 1 ]; then
181        return 0
182    fi
183
184    if [ -n "${pkg}" ]; then
185        if ! make O="${dir}" "${pkg}-dirclean" >> "${dir}/logfile" 2>&1; then
186            return 2
187        fi
188    fi
189
190    # shellcheck disable=SC2086
191    if ! BR_FORCE_CHECK_DEPENDENCIES=YES make O="${dir}" ${pkg} >> "${dir}/logfile" 2>&1; then
192        return 2
193    fi
194
195    # legal-info done systematically, because some packages have different
196    # sources depending on the configuration (e.g. lua-5.2 vs. lua-5.3)
197    if ! make O="${dir}" legal-info >> "${dir}/logfile" 2>&1; then
198        return 3
199    fi
200
201    # Validate that we generate proper json as show-info
202    { tput smso; printf '>>> Running show-info\n'; tput rmso; } >> "${dir}/logfile" 2> /dev/null;
203    JQ="$(which jq 2> /dev/null)"
204    if [ -z "${JQ}" ]; then
205        make O="${dir}" host-jq >> "${dir}/logfile" 2>&1
206        JQ="${dir}/host/bin/jq"
207    fi
208    if ! make O="${dir}" "${pkg:+${pkg}-}show-info" > "${dir}/info.json" 2>> "${dir}/logfile"; then
209        return 4
210    fi
211    if ! "${JQ}" . < "${dir}/info.json" >> "${dir}/logfile" 2>&1; then
212        return 4
213    fi
214
215    # If we get here, the build was successful. Clean up the build/host
216    # directories to save disk space, unless 'keep' was set.
217    if [ ${keep} -ne 1 ]; then
218        make O="${dir}" clean >> "${dir}/logfile" 2>&1
219    fi
220}
221
222help() {
223    cat <<_EOF_
224test-pkg: test-build a package against various toolchains and architectures
225
226The supplied config snippet is appended to each toolchain config, the
227resulting configuration is checked to ensure it still contains all options
228specified in the snippet; if any is missing, the build is skipped, on the
229assumption that the package under test requires a toolchain or architecture
230feature that is missing.
231
232In case failures are noticed, you can fix the package and just re-run the
233same command again; it will re-run the test where it failed. If you did
234specify a package (with -p), the package build dir will be removed first.
235
236The list of toolchains is retrieved from ${TOOLCHAINS_CSV}.
237Only the external toolchains are tried, because building a Buildroot toolchain
238would take too long. An alternative toolchains CSV file can be specified with
239the -t option. This file should have lines consisting of the path to the
240toolchain config fragment and the required host architecture, separated by a
241comma. The config fragments should contain only the toolchain and architecture
242settings.
243
244By default, a useful subset of toolchains is tested. If needed, all
245toolchains can be tested (-a), an arbitrary number of toolchains (-n
246in order, -r for random).
247
248Options:
249
250    -h, --help
251        Print this help.
252
253    -c CFG, --config-snippet CFG
254        Use the CFG file as the source for the config snippet. This file
255        should contain all the config options required to build a package.
256
257    -d DIR, --build-dir DIR
258        Do the builds in directory DIR, one sub-dir per toolchain.
259        If not specified, defaults to \${HOME}/br-test-pkg
260
261    -p PKG, --package PKG
262        Test-build the package PKG, by running 'make PKG'; if not specified,
263        just runs 'make'.
264
265    -a, --all
266        Test all toolchains, instead of the default subset defined by
267        Buildroot developers.
268
269    -n N, --number N
270        Test N toolchains, in the order defined in the toolchain CSV
271        file.
272
273    -r N, --random N
274        Limit the tests to the N randomly selected toolchains.
275
276    -t CSVFILE, --toolchains-csv CSVFILE
277        CSV file containing the paths to config fragments of toolchains to
278        try. If not specified, the toolchains in ${TOOLCHAINS_CSV} will be
279        used.
280
281    -k, --keep
282        Keep the build directories even if the build succeeds.
283        Note: the logfile and configuration is always retained, even without
284        this option.
285
286    --prepare-only
287        Only prepare the .config files, but do not build them. Output the
288        list of build directories to stdout, and the status on stderr.
289
290Example:
291
292    Testing libcec would require a config snippet that contains:
293        BR2_PACKAGE_LIBCEC=y
294
295    Testing libcurl with openSSL support would require a snippet such as:
296        BR2_PACKAGE_OPENSSL=y
297        BR2_PACKAGE_LIBCURL=y
298
299_EOF_
300}
301
302my_name="${0##*/}"
303main "${@}"
304