1#!/bin/bash
2#
3# Yocto meta virtualization build and run script
4#
5# This script is building Yocto xen-image-minimal for qemu targets and run
6# them using runqemu inside yocto to check that dom0 is booting properly.
7# The build is using a local xen source tree so that specific patches can be
8# tested.
9# In order to optimize the build time, a build cache is used so that only xen
10# packages and its dependencies are rebuilt (qemu and final image mainly).
11#
12# get command error even when piped.
13set -o pipefail
14
15# Directories
16YOCTODIR="$HOME/yocto-layers"
17CACHEDIR="$HOME/yocto-cache"
18LOGDIR="$HOME/logs"
19XENDIR="$HOME/xen"
20BUILDDIR="$HOME/build"
21OUTPUTDIR=`pwd`/binaries
22
23# what yocto bsp we support
24TARGET_SUPPORTED="qemuarm qemuarm64 qemux86-64"
25VERBOSE="n"
26TARGETLIST=""
27BUILDJOBS="8"
28IMAGE_FMT=""
29
30# actions to do
31do_clean="n"
32do_build="y"
33do_run="y"
34do_localsrc="n"
35do_dump="n"
36do_copy="n"
37build_result=0
38
39# layers to include in the project
40build_layerlist="poky/meta poky/meta-poky poky/meta-yocto-bsp \
41                 meta-openembedded/meta-oe meta-openembedded/meta-python \
42                 meta-openembedded/meta-networking \
43                 meta-openembedded/meta-filesystems \
44                 meta-virtualization"
45
46# yocto image to build
47build_image="xen-image-minimal"
48
49function print_progress() {
50    echo -n "$(date +%T) $*"
51}
52
53function run_task() {
54    local task_name="$1"
55    local task_target="$2"
56
57    task_log="${task_name//project_}-${task_target}"
58
59    mkdir -p "${LOGDIR}"
60    print_progress
61    echo -n "${task_name//project_} ${task_target}: "
62    if [ "${VERBOSE}" = "n" ]; then
63        "$@" > "${LOGDIR}/${task_log}.log" 2>&1
64    else
65        "$@" 2>&1 | tee "${LOGDIR}/${task_log}.log"
66    fi
67
68    if [ ${?} -ne 0 ]; then
69        echo "Error"
70        build_result=$((build_result+1))
71        if [ "${do_dump}" = "y" ]; then
72            echo
73            echo "############ LOGS-START ############"
74            cat "${LOGDIR}/${task_log}.log"
75            echo "############  LOGS-END  ############"
76            echo
77        fi
78        return 1
79    else
80        echo "OK"
81        return 0
82    fi
83}
84
85function project_create() {
86    target="${1:?}"
87    destdir="${BUILDDIR}/${target}"
88
89    (
90        # init yocto project
91        source "${YOCTODIR}/poky/oe-init-build-env" "${destdir}"
92
93        # add needed layers
94        for layer in ${build_layerlist}; do
95            bitbake-layers add-layer "${YOCTODIR}/${layer}" || exit 1
96        done
97    ) || return 1
98
99    # Detect latest version available in Yocto and use it instead of default
100    # one.
101    XENVERS=$(grep -e "^XEN_REL" \
102        "${YOCTODIR}"/meta-virtualization/recipes-extended/xen/xen_*.bb \
103        2> /dev/null | tr -d ' ' | tr -d '?' | tr -d '"' \
104        | sed -e "s/.*=//" | sort -V | tail -n 1)
105
106    # customize project configuration
107    cat <<EOF >> "${destdir}/conf/local.conf"
108# Yocto BSP
109MACHINE = "${target}"
110
111# Use local cache to reuse previous builds results
112SSTATE_DIR = "${CACHEDIR}/sstate-cache"
113DL_DIR = "${CACHEDIR}/downloads"
114
115# Enable xen and virtualization
116DISTRO_FEATURES = " virtualization xen ipv4"
117
118# Speed up run by not generating ssh host keys
119IMAGE_INSTALL:append:pn-xen-image-minimal = " ssh-pregen-hostkeys"
120
121# Save some disk space
122INHERIT += "rm_work"
123
124# Reduce number of jobs
125BB_NUMBER_THREADS="${BUILDJOBS}"
126
127# Use latest Xen version
128PREFERRED_VERSION:pn-xen = "${XENVERS}%"
129PREFERRED_VERSION:pn-xen-tools = "${XENVERS}%"
130
131# Use autorev for now as Xen SHA used by latest yocto recipe for Xen does not
132# include fixes required to build x86 on arm
133SRCREV:pn-xen = "\${AUTOREV}"
134SRCREV:pn-xen-tools = "\${AUTOREV}"
135
136# Disable all QA errors as the recipe is not up to date with changes in Xen
137# when we use local sources
138ERROR_QA:pn-xen = "arch"
139ERROR_QA:pn-xen-tools = "arch"
140
141EOF
142
143    if [ "${do_localsrc}" = "y" ]; then
144        XENBASE=$(dirname "$(realpath -m "${XENDIR}")")
145        XENSUB=$(basename "$(realpath -m "${XENDIR}")")
146
147        cat <<EOF >> "${destdir}/conf/local.conf"
148# Use local sources for xen and xen-tools
149FILESEXTRAPATHS:prepend:pn-xen := "${XENBASE}:"
150FILESEXTRAPATHS:prepend:pn-xen-tools := "${XENBASE}:"
151
152SRC_URI:pn-xen = "file://${XENSUB}/;subdir=local-xen/"
153SRC_URI:pn-xen-tools = "file://${XENSUB}/;subdir=local-xen/"
154
155S:pn-xen = "\${WORKDIR}/local-xen/${XENSUB}"
156S:pn-xen-tools = "\${WORKDIR}/local-xen/${XENSUB}"
157
158SRCPV:pn-xen = "1"
159SRCPV:pn-xen-tools = "1"
160
161EOF
162    fi
163}
164
165function project_build() {
166    target="${1:?}"
167    destdir="${BUILDDIR}/${target}"
168
169    (
170        source "${YOCTODIR}/poky/oe-init-build-env" "${destdir}"
171
172        bitbake "${build_image}" || exit 1
173        if [ $do_copy = "y" ]
174        then
175            if [ $target = "qemuarm" ]
176            then
177                mkdir -p $OUTPUTDIR
178                cp $BUILDDIR/tmp/deploy/images/qemuarm/zImage $OUTPUTDIR
179                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-qemuarm $OUTPUTDIR
180                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-image-minimal-qemuarm.rootfs.tar.bz2 $OUTPUTDIR
181            fi
182        fi
183    ) || return 1
184}
185
186function project_clean() {
187    target="${1:?}"
188    destdir="${BUILDDIR}/${target}"
189
190    rm -rf "${destdir}"
191}
192
193function project_run() {
194    target="${1:?}"
195    destdir="${BUILDDIR}/${target}"
196    (
197        source "${YOCTODIR}/poky/oe-init-build-env" "${destdir}" > /dev/null 2>&1
198
199        /usr/bin/expect <<EOF
200set timeout 1000
201spawn bash -c "runqemu serialstdio nographic slirp ${IMAGE_FMT}"
202
203expect_after {
204    -re "(.*)\r" {
205        exp_continue
206    }
207    timeout {send_user "ERROR-Timeout!\n"; exit 1}
208    eof {send_user "ERROR-EOF!\n"; exit 1}
209}
210
211# wait initial login
212expect -re ".* login: "
213send "root\r"
214expect -re "root@.*# "
215
216EOF
217    exit $?
218    ) || return 1
219}
220
221function help() {
222    cat <<EOF
223Usage: ${0} [TARGET1] [TARGET2]
224
225This script is build the yocto xen-image-minimal for different qemu targets
226and is running it after.
227Without any target specified, all supported targets are done.
228
229Options:
230  -h, --help       Print this help
231  -v, --verbose    Verbose build
232  --list-target    List supported targets
233  --clean          Clean existing project before starting
234  --no-build       Do not build (to run an already built project)
235  --no-run         Do not run
236  --num-jobs=NUM   Define the number of parallel jobs in Yocto.
237                   Default: ${BUILDJOBS}
238  --dump-log       On error, dump the logs on the console
239  --image=IMG      Yocto image or package to build
240                   Default: xen-image-minimal
241  --xen-dir=DIR    path to xen hypervisor source tree
242                   if not provide, normal yocto version of xen is built
243                   Default: ${XENDIR}
244  --out-dir=DIR    directory where to create the projectss
245                   Default: ${BUILDDIR}
246  --log-dir=DIR    directory to store logs
247                   Default: ${LOGDIR}
248  --cache-dir=DIR  directory where to take and store build cache
249                   Default: ${CACHEDIR}
250  --layer-dir=DIR  directory containing the checkout of yocto layers
251                   Default: ${YOCTODIR}
252  --copy-output    Copy output binaries to binaries/
253EOF
254}
255
256for OPTION in "$@"
257do
258    case ${OPTION} in
259        -h|--help)
260            help
261            exit 0
262            ;;
263        -v|--verbose)
264            VERBOSE="y"
265            ;;
266        --list-targets)
267            echo "${TARGET_SUPPORTED}"
268            exit 0
269            ;;
270        --clean)
271            do_clean="y"
272            ;;
273        --no-build)
274            do_build="n"
275            ;;
276        --no-run)
277            do_run="n"
278            ;;
279        --dump-log)
280            do_dump="y"
281            ;;
282        --num-jobs=*)
283            BUILDJOBS="${OPTION#*=}"
284            ;;
285        --image=*)
286            build_image="${OPTION#*=}"
287            ;;
288        --xen-dir=*)
289            XENDIR="${OPTION#*=}"
290            if [ ! -e "${XENDIR}/xen/Makefile" ]; then
291                echo "No Xen source tree in ${XENDIR}"
292                exit 1
293            fi
294            do_localsrc="y"
295            ;;
296        --out-dir=*)
297            BUILDDIR="${OPTION#*=}"
298            ;;
299        --log-dir=*)
300            LOGDIR="${OPTION#*=}"
301            ;;
302        --cache-dir=*)
303            CACHEDIR="${OPTION#*=}"
304            ;;
305        --layer-dir=*)
306            YOCTODIR="${OPTION#*=}"
307            ;;
308        --copy-output)
309            do_copy="y"
310            ;;
311        --*)
312            echo "Invalid option ${OPTION}"
313            help
314            exit 1
315            ;;
316        *)
317            if echo "${TARGET_SUPPORTED}" | grep -q -w "${OPTION}"; then
318                TARGETLIST="${TARGETLIST} ${OPTION}"
319            else
320                echo "Unsupported target ${OPTION}"
321                exit 1
322            fi
323            ;;
324    esac
325done
326
327# if no target is specified build all targets
328if [ -z "${TARGETLIST}" ]; then
329    TARGETLIST="${TARGET_SUPPORTED}"
330fi
331
332mkdir -p "${CACHEDIR}"
333mkdir -p "${LOGDIR}"
334mkdir -p "${BUILDDIR}"
335
336# Make sure we have an absolute path
337YOCTODIR=$(realpath -m "${YOCTODIR}")
338CACHEDIR=$(realpath -m "${CACHEDIR}")
339BUILDDIR=$(realpath -m "${BUILDDIR}")
340LOGDIR=$(realpath -m "${LOGDIR}")
341if [ "${do_localsrc}" = "y" ]; then
342    XENDIR=$(realpath -m "${XENDIR}")
343fi
344
345# Check that we have all the layers we need
346for f in ${build_layerlist}; do
347    if [ ! -f "${YOCTODIR}/${f}/conf/layer.conf" ]; then
348        echo "Layer ${f} missing in ${YOCTODIR}"
349        exit 1
350    fi
351done
352
353for f in ${TARGETLIST}; do
354    if [ "${do_clean}" = "y" ]; then
355        run_task project_clean "${f}"
356    fi
357    if [ ! -f "${BUILDDIR}/${f}/conf/local.conf" ]; then
358        run_task project_create "${f}"
359    fi
360    if [ -f "${BUILDDIR}/${f}/conf/local.conf" ]; then
361        # Set the right image target
362        if [ "$f" = "qemux86-64" ]; then
363            IMAGE_FMT=""
364        else
365            IMAGE_FMT="ext4"
366        fi
367
368        if [ "${do_build}" = "y" ]; then
369            run_task project_build "${f}"
370        fi
371        if [ "${do_run}" = "y" ]; then
372            run_task project_run "${f}"
373        fi
374
375    fi
376done
377
378print_progress "Build Complete (${build_result} errors)"
379echo
380exit ${build_result}
381
382