1#!/bin/bash
2#
3# Yocto meta virtualization build and run script
4#
5# This script is building Yocto xen-image-minimal for qemu targets and run
6# them using runqemu inside yocto to check that dom0 is booting properly.
7# The build is using a local xen source tree so that specific patches can be
8# tested.
9# In order to optimize the build time, a build cache is used so that only xen
10# packages and its dependencies are rebuilt (qemu and final image mainly).
11#
12# get command error even when piped.
13set -o pipefail
14
15# Directories
16YOCTODIR="$HOME/yocto-layers"
17CACHEDIR="$HOME/yocto-cache"
18LOGDIR="$HOME/logs"
19XENDIR="$HOME/xen"
20BUILDDIR="$HOME/build"
21OUTPUTDIR=`pwd`/binaries
22
23# what yocto bsp we support
24TARGET_SUPPORTED="qemuarm qemuarm64 qemux86-64"
25VERBOSE="n"
26TARGETLIST=""
27BUILDJOBS="8"
28
29# actions to do
30do_clean="n"
31do_build="y"
32do_run="y"
33do_localsrc="n"
34do_dump="n"
35do_copy="n"
36build_result=0
37
38# layers to include in the project
39build_layerlist="poky/meta poky/meta-poky poky/meta-yocto-bsp \
40                 meta-openembedded/meta-oe meta-openembedded/meta-python \
41                 meta-openembedded/meta-filesystems \
42                 meta-openembedded/meta-networking meta-virtualization"
43
44# yocto image to build
45build_image="xen-image-minimal"
46
47function print_progress() {
48    echo -n "$(date +%T) $*"
49}
50
51function run_task() {
52    local task_name="$1"
53    local task_target="$2"
54
55    task_log="${task_name//project_}-${task_target}"
56
57    mkdir -p "${LOGDIR}"
58    print_progress
59    echo -n "${task_name//project_} ${task_target}: "
60    if [ "${VERBOSE}" = "n" ]; then
61        "$@" > "${LOGDIR}/${task_log}.log" 2>&1
62    else
63        "$@" 2>&1 | tee "${LOGDIR}/${task_log}.log"
64    fi
65
66    if [ ${?} -ne 0 ]; then
67        echo "Error"
68        build_result=$((build_result+1))
69        if [ "${do_dump}" = "y" ]; then
70            echo
71            echo "############ LOGS-START ############"
72            cat "${LOGDIR}/${task_log}.log"
73            echo "############  LOGS-END  ############"
74            echo
75        fi
76        return 1
77    else
78        echo "OK"
79        return 0
80    fi
81}
82
83function project_create() {
84    target="${1:?}"
85    destdir="${BUILDDIR}/${target}"
86
87    (
88        # init yocto project
89        source "${YOCTODIR}/poky/oe-init-build-env" "${destdir}"
90
91        # add needed layers
92        for layer in ${build_layerlist}; do
93            bitbake-layers add-layer "${YOCTODIR}/${layer}" || exit 1
94        done
95    ) || return 1
96
97    # Detect latest version available in Yocto and use it instead of default
98    # one.
99    XENVERS=$(grep -e "^XEN_REL" \
100        "${YOCTODIR}"/meta-virtualization/recipes-extended/xen/xen_*.bb \
101        2> /dev/null | tr -d ' ' | tr -d '?' | tr -d '"' \
102        | sed -e "s/.*=//" | sort -V | tail -n 1)
103
104    # customize project configuration
105    cat <<EOF >> "${destdir}/conf/local.conf"
106# Yocto BSP
107MACHINE = "${target}"
108
109# Use local cache to reuse previous builds results
110SSTATE_DIR = "${CACHEDIR}/sstate-cache"
111DL_DIR = "${CACHEDIR}/downloads"
112
113# Enable xen and virtualization
114DISTRO_FEATURES = " virtualization xen ipv4"
115
116# Speed up run by not generating ssh host keys
117IMAGE_INSTALL:append:pn-xen-image-minimal = " ssh-pregen-hostkeys"
118
119# Save some disk space
120INHERIT += "rm_work"
121
122# Reduce number of jobs
123BB_NUMBER_THREADS="${BUILDJOBS}"
124
125# Use latest Xen version
126PREFERRED_VERSION:pn-xen = "${XENVERS}%"
127PREFERRED_VERSION:pn-xen-tools = "${XENVERS}%"
128
129# Use autorev for now as Xen SHA used by latest yocto recipe for Xen does not
130# include fixes required to build x86 on arm
131SRCREV:pn-xen = "\${AUTOREV}"
132SRCREV:pn-xen-tools = "\${AUTOREV}"
133
134# Disable all QA errors as the recipe is not up to date with changes in Xen
135# when we use local sources
136ERROR_QA:pn-xen = "arch"
137ERROR_QA:pn-xen-tools = "arch"
138
139EOF
140
141    if [ "${do_localsrc}" = "y" ]; then
142        XENBASE=$(dirname "$(realpath -m "${XENDIR}")")
143        XENSUB=$(basename "$(realpath -m "${XENDIR}")")
144
145        cat <<EOF >> "${destdir}/conf/local.conf"
146# Use local sources for xen and xen-tools
147FILESEXTRAPATHS:prepend:pn-xen := "${XENBASE}:"
148FILESEXTRAPATHS:prepend:pn-xen-tools := "${XENBASE}:"
149
150SRC_URI:pn-xen = "file://${XENSUB}/;subdir=local-xen/"
151SRC_URI:pn-xen-tools = "file://${XENSUB}/;subdir=local-xen/"
152
153S:pn-xen = "\${WORKDIR}/local-xen/${XENSUB}"
154S:pn-xen-tools = "\${WORKDIR}/local-xen/${XENSUB}"
155
156SRCPV:pn-xen = "1"
157SRCPV:pn-xen-tools = "1"
158
159EOF
160    fi
161}
162
163function project_build() {
164    target="${1:?}"
165    destdir="${BUILDDIR}/${target}"
166
167    (
168        source "${YOCTODIR}/poky/oe-init-build-env" "${destdir}"
169
170        bitbake "${build_image}" || exit 1
171        if [ $do_copy = "y" ]
172        then
173            if [ $target = "qemuarm" ]
174            then
175                mkdir -p $OUTPUTDIR
176                cp $BUILDDIR/tmp/deploy/images/qemuarm/zImage $OUTPUTDIR
177                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-qemuarm $OUTPUTDIR
178                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-image-minimal-qemuarm.tar.bz2 $OUTPUTDIR
179            fi
180        fi
181    ) || return 1
182}
183
184function project_clean() {
185    target="${1:?}"
186    destdir="${BUILDDIR}/${target}"
187
188    rm -rf "${destdir}"
189}
190
191function project_run() {
192    target="${1:?}"
193    destdir="${BUILDDIR}/${target}"
194    (
195        source "${YOCTODIR}/poky/oe-init-build-env" "${destdir}" > /dev/null 2>&1
196
197        /usr/bin/expect <<EOF
198set timeout 1000
199spawn bash -c "runqemu serialstdio nographic slirp"
200
201expect_after {
202    -re "(.*)\r" {
203        exp_continue
204    }
205    timeout {send_user "ERROR-Timeout!\n"; exit 1}
206    eof {send_user "ERROR-EOF!\n"; exit 1}
207}
208
209# wait initial login
210expect -re ".* login: "
211send "root\r"
212expect -re "root@.*# "
213
214EOF
215    exit $?
216    ) || return 1
217}
218
219function help() {
220    cat <<EOF
221Usage: ${0} [TARGET1] [TARGET2]
222
223This script is build the yocto xen-image-minimal for different qemu targets
224and is running it after.
225Without any target specified, all supported targets are done.
226
227Options:
228  -h, --help       Print this help
229  -v, --verbose    Verbose build
230  --list-target    List supported targets
231  --clean          Clean existing project before starting
232  --no-build       Do not build (to run an already built project)
233  --no-run         Do not run
234  --num-jobs=NUM   Define the number of parallel jobs in Yocto.
235                   Default: ${BUILDJOBS}
236  --dump-log       On error, dump the logs on the console
237  --image=IMG      Yocto image or package to build
238                   Default: xen-image-minimal
239  --xen-dir=DIR    path to xen hypervisor source tree
240                   if not provide, normal yocto version of xen is built
241                   Default: ${XENDIR}
242  --out-dir=DIR    directory where to create the projectss
243                   Default: ${BUILDDIR}
244  --log-dir=DIR    directory to store logs
245                   Default: ${LOGDIR}
246  --cache-dir=DIR  directory where to take and store build cache
247                   Default: ${CACHEDIR}
248  --layer-dir=DIR  directory containing the checkout of yocto layers
249                   Default: ${YOCTODIR}
250  --copy-output    Copy output binaries to binaries/
251EOF
252}
253
254for OPTION in "$@"
255do
256    case ${OPTION} in
257        -h|--help)
258            help
259            exit 0
260            ;;
261        -v|--verbose)
262            VERBOSE="y"
263            ;;
264        --list-targets)
265            echo "${TARGET_SUPPORTED}"
266            exit 0
267            ;;
268        --clean)
269            do_clean="y"
270            ;;
271        --no-build)
272            do_build="n"
273            ;;
274        --no-run)
275            do_run="n"
276            ;;
277        --dump-log)
278            do_dump="y"
279            ;;
280        --num-jobs=*)
281            BUILDJOBS="${OPTION#*=}"
282            ;;
283        --image=*)
284            build_image="${OPTION#*=}"
285            ;;
286        --xen-dir=*)
287            XENDIR="${OPTION#*=}"
288            if [ ! -e "${XENDIR}/xen/Makefile" ]; then
289                echo "No Xen source tree in ${XENDIR}"
290                exit 1
291            fi
292            do_localsrc="y"
293            ;;
294        --out-dir=*)
295            BUILDDIR="${OPTION#*=}"
296            ;;
297        --log-dir=*)
298            LOGDIR="${OPTION#*=}"
299            ;;
300        --cache-dir=*)
301            CACHEDIR="${OPTION#*=}"
302            ;;
303        --layer-dir=*)
304            YOCTODIR="${OPTION#*=}"
305            ;;
306        --copy-output)
307            do_copy="y"
308            ;;
309        --*)
310            echo "Invalid option ${OPTION}"
311            help
312            exit 1
313            ;;
314        *)
315            if echo "${TARGET_SUPPORTED}" | grep -q -w "${OPTION}"; then
316                TARGETLIST="${TARGETLIST} ${OPTION}"
317            else
318                echo "Unsupported target ${OPTION}"
319                exit 1
320            fi
321            ;;
322    esac
323done
324
325# if no target is specified build all targets
326if [ -z "${TARGETLIST}" ]; then
327    TARGETLIST="${TARGET_SUPPORTED}"
328fi
329
330mkdir -p "${CACHEDIR}"
331mkdir -p "${LOGDIR}"
332mkdir -p "${BUILDDIR}"
333
334# Make sure we have an absolute path
335YOCTODIR=$(realpath -m "${YOCTODIR}")
336CACHEDIR=$(realpath -m "${CACHEDIR}")
337BUILDDIR=$(realpath -m "${BUILDDIR}")
338LOGDIR=$(realpath -m "${LOGDIR}")
339if [ "${do_localsrc}" = "y" ]; then
340    XENDIR=$(realpath -m "${XENDIR}")
341fi
342
343# Check that we have all the layers we need
344for f in ${build_layerlist}; do
345    if [ ! -f "${YOCTODIR}/${f}/conf/layer.conf" ]; then
346        echo "Layer ${f} missing in ${YOCTODIR}"
347        exit 1
348    fi
349done
350
351for f in ${TARGETLIST}; do
352    if [ "${do_clean}" = "y" ]; then
353        run_task project_clean "${f}"
354    fi
355    if [ ! -f "${BUILDDIR}/${f}/conf/local.conf" ]; then
356        run_task project_create "${f}"
357    fi
358    if [ -f "${BUILDDIR}/${f}/conf/local.conf" ]; then
359        if [ "${do_build}" = "y" ]; then
360            run_task project_build "${f}"
361        fi
362        if [ "${do_run}" = "y" ]; then
363            run_task project_run "${f}"
364        fi
365
366    fi
367done
368
369print_progress "Build Complete (${build_result} errors)"
370echo
371exit ${build_result}
372
373