1#!/usr/bin/env bash
2
3# This script is a wrapper to the other download backends.
4# Its role is to ensure atomicity when saving downloaded files
5# back to BR2_DL_DIR, and not clutter BR2_DL_DIR with partial,
6# failed downloads.
7
8# To avoid cluttering BR2_DL_DIR, we download to a trashable
9# location, namely in $(BUILD_DIR).
10# Then, we move the downloaded file to a temporary file in the
11# same directory as the final output file.
12# This allows us to finally atomically rename it to its final
13# name.
14# If anything goes wrong, we just remove all the temporaries
15# created so far.
16
17# We want to catch any unexpected failure, and exit immediately.
18set -e
19
20export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:lru:qf:e"
21
22main() {
23    local OPT OPTARG
24    local backend output large_file recurse quiet rc
25    local -a uris hfiles
26
27    # Parse our options; anything after '--' is for the backend
28    while getopts ":c:d:D:o:n:N:H:lrf:u:qp:" OPT; do
29        case "${OPT}" in
30        c)  cset="${OPTARG}";;
31        d)  dl_dir="${OPTARG}";;
32        D)  old_dl_dir="${OPTARG}";;
33        o)  output="${OPTARG}";;
34        n)  raw_base_name="${OPTARG}";;
35        N)  base_name="${OPTARG}";;
36        H)  hfiles+=( "${OPTARG}" );;
37        l)  large_file="-l";;
38        r)  recurse="-r";;
39        f)  filename="${OPTARG}";;
40        u)  uris+=( "${OPTARG}" );;
41        p)  post_process="${OPTARG}";;
42        q)  quiet="-q";;
43        :)  error "option '%s' expects a mandatory argument\n" "${OPTARG}";;
44        \?) error "unknown option '%s'\n" "${OPTARG}";;
45        esac
46    done
47
48    # Forget our options, and keep only those for the backend
49    shift $((OPTIND-1))
50
51    if [ -z "${output}" ]; then
52        error "no output specified, use -o\n"
53    fi
54
55    # Legacy handling: check if the file already exists in the global
56    # download directory. If it does, hard-link it. If it turns out it
57    # was an incorrect download, we'd still check it below anyway.
58    # If we can neither link nor copy, fallback to doing a download.
59    # NOTE! This is not atomic, is subject to TOCTTOU, but the whole
60    # dl-wrapper runs under an flock, so we're safe.
61    if [ ! -e "${output}" -a -e "${old_dl_dir}/${filename}" ]; then
62        ln "${old_dl_dir}/${filename}" "${output}" || \
63        cp "${old_dl_dir}/${filename}" "${output}" || \
64        true
65    fi
66
67    # If the output file already exists and:
68    # - there's no .hash file: do not download it again and exit promptly
69    # - matches all its hashes: do not download it again and exit promptly
70    # - fails at least one of its hashes: force a re-download
71    # - there's no hash (but a .hash file): consider it a hard error
72    if [ -e "${output}" ]; then
73        if support/download/check-hash ${quiet} "${output}" "${output##*/}" "${hfiles[@]}"; then
74            exit 0
75        elif [ ${?} -ne 2 ]; then
76            # Do not remove the file, otherwise it might get re-downloaded
77            # from a later location (i.e. primary -> upstream -> mirror).
78            # Do not print a message, check-hash already did.
79            exit 1
80        fi
81        rm -f "${output}"
82        warn "Re-downloading '%s'...\n" "${output##*/}"
83    fi
84
85    # Look through all the uris that we were given to download the package
86    # source
87    download_and_check=0
88    rc=1
89    for uri in "${uris[@]}"; do
90        backend_urlencode="${uri%%+*}"
91        backend="${backend_urlencode%|*}"
92        case "${backend}" in
93            git|svn|cvs|bzr|file|scp|hg|sftp) ;;
94            *) backend="wget" ;;
95        esac
96        uri=${uri#*+}
97
98        urlencode=${backend_urlencode#*|}
99        # urlencode must be "urlencode"
100        [ "${urlencode}" != "urlencode" ] && urlencode=""
101
102        # tmpd is a temporary directory in which backends may store
103        # intermediate by-products of the download.
104        # tmpf is the file in which the backends should put the downloaded
105        # content.
106        # tmpd is located in $(BUILD_DIR), so as not to clutter the (precious)
107        # $(BR2_DL_DIR)
108        # We let the backends create tmpf, so they are able to set whatever
109        # permission bits they want (although we're only really interested in
110        # the executable bit.)
111        tmpd="$(mktemp -d "${BUILD_DIR}/.${output##*/}.XXXXXX")"
112        tmpf="${tmpd}/output"
113
114        # Helpers expect to run in a directory that is *really* trashable, so
115        # they are free to create whatever files and/or sub-dirs they might need.
116        # Doing the 'cd' here rather than in all backends is easier.
117        cd "${tmpd}"
118
119        # If the backend fails, we can just remove the content of the temporary
120        # directory to remove all the cruft it may have left behind, and try
121        # the next URI until it succeeds. Once out of URI to try, we need to
122        # cleanup and exit.
123        if ! "${OLDPWD}/support/download/${backend}" \
124                $([ -n "${urlencode}" ] && printf %s '-e') \
125                -c "${cset}" \
126                -d "${dl_dir}" \
127                -n "${raw_base_name}" \
128                -N "${base_name}" \
129                -f "${filename}" \
130                -u "${uri}" \
131                -o "${tmpf}" \
132                ${quiet} ${large_file} ${recurse} -- "${@}"
133        then
134            # cd back to keep path coherence
135            cd "${OLDPWD}"
136            rm -rf "${tmpd}"
137            continue
138        fi
139
140        if [ -n "${post_process}" ] ; then
141            if ! "${OLDPWD}/support/download/${post_process}-post-process" \
142                    -o "${tmpf}" \
143                    -n "${raw_base_name}"
144            then
145                # cd back to keep path coherence
146                cd "${OLDPWD}"
147                rm -rf "${tmpd}"
148                continue
149            fi
150        fi
151
152        # cd back to free the temp-dir, so we can remove it later
153        cd "${OLDPWD}"
154
155        # Check if the downloaded file is sane, and matches the stored hashes
156        # for that file
157        if support/download/check-hash ${quiet} "${tmpf}" "${output##*/}" "${hfiles[@]}"; then
158            rc=0
159        else
160            if [ ${?} -ne 3 ]; then
161                rm -rf "${tmpd}"
162                continue
163            fi
164
165            # the hash file exists and there was no hash to check the file
166            # against
167            rc=1
168        fi
169        download_and_check=1
170        break
171    done
172
173    # We tried every URI possible, none seems to work or to check against the
174    # available hash. *ABORT MISSION*
175    if [ "${download_and_check}" -eq 0 ]; then
176        rm -rf "${tmpd}"
177        exit 1
178    fi
179
180    # tmp_output is in the same directory as the final output, so we can
181    # later move it atomically.
182    tmp_output="$(mktemp "${output}.XXXXXX")"
183
184    # 'mktemp' creates files with 'go=-rwx', so the files are not accessible
185    # to users other than the one doing the download (and root, of course).
186    # This can be problematic when a shared BR2_DL_DIR is used by different
187    # users (e.g. on a build server), where all users may write to the shared
188    # location, since other users would not be allowed to read the files
189    # another user downloaded.
190    # So, we restore the 'go' access rights to a more sensible value, while
191    # still abiding by the current user's umask. We must do that before the
192    # final 'mv', so just do it now.
193    # Some backends (cp and scp) may create executable files, so we need to
194    # carry the executable bit if needed.
195    [ -x "${tmpf}" ] && new_mode=755 || new_mode=644
196    new_mode=$(printf "%04o" $((0${new_mode} & ~0$(umask))))
197    chmod ${new_mode} "${tmp_output}"
198
199    # We must *not* unlink tmp_output, otherwise there is a small window
200    # during which another download process may create the same tmp_output
201    # name (very, very unlikely; but not impossible.)
202    # Using 'cp' is not reliable, since 'cp' may unlink the destination file
203    # if it is unable to open it with O_WRONLY|O_TRUNC; see:
204    #   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/cp.html
205    # Since the destination filesystem can be anything, it might not support
206    # O_TRUNC, so 'cp' would unlink it first.
207    # Use 'cat' and append-redirection '>>' to save to the final location,
208    # since that is the only way we can be 100% sure of the behaviour.
209    if ! cat "${tmpf}" >>"${tmp_output}"; then
210        rm -rf "${tmpd}" "${tmp_output}"
211        exit 1
212    fi
213    rm -rf "${tmpd}"
214
215    # tmp_output and output are on the same filesystem, so POSIX guarantees
216    # that 'mv' is atomic, because it then uses rename() that POSIX mandates
217    # to be atomic, see:
218    #   http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html
219    if ! mv -f "${tmp_output}" "${output}"; then
220        rm -f "${tmp_output}"
221        exit 1
222    fi
223
224    return ${rc}
225}
226
227trace()  { local msg="${1}"; shift; printf "%s: ${msg}" "${my_name}" "${@}"; }
228warn()   { trace "${@}" >&2; }
229errorN() { local ret="${1}"; shift; warn "${@}"; exit ${ret}; }
230error()  { errorN 1 "${@}"; }
231
232my_name="${0##*/}"
233main "${@}"
234