1#!/usr/bin/env bash
2
3# Copyright 2018 The Fuchsia Authors
4#
5# Use of this source code is governed by a MIT-style
6# license that can be found in the LICENSE file or at
7# https://opensource.org/licenses/MIT
8
9readonly SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
10readonly ZIRCON_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
11readonly PREBUILTS_DIR="$(cd "${ZIRCON_ROOT}/prebuilt" && pwd)"
12readonly DOWNLOAD_DIR="${PREBUILTS_DIR}/downloads"
13readonly ENSURE_FILE="${PREBUILTS_DIR}/zircon.ensure"
14readonly VERSIONS_FILE="${PREBUILTS_DIR}/zircon.versions"
15readonly URL_PREFIX="https://chrome-infra-packages.appspot.com/dl/fuchsia"
16
17# This script assumes that ENSURE_FILE and VERSIONS_FILE match up.
18# `cipd ensure` checks this and `cips ensure-file-resolve` ensures it.
19# When a `cipd` binary is available, ENSURE_FILE controls the downloads
20# and points `cipd` at VERSIONS_FILE for resolved pinned versions.
21# Otherwise, VERSIONS_FILE controls the downloads directly.  In both
22# cases we write $DOWNLOAD_DIR/$PACKAGE.stamp files with the versions
23# from VERSIONS_FILE on faith that that's what we just unpacked.
24
25set -o pipefail
26
27cipd_ok=true
28case "$#:$1" in
290:)
30  mode=update
31  ;;
321:--verify)
33  mode=verify
34  ;;
351:--list)
36  mode=list
37  ;;
381:--resolve)
39  mode=resolve
40  ;;
411:--no-cipd)
42  mode=update
43  cipd_ok=false
44  ;;
45*)
46  echo >&2 "Usage: $0 [--verify|--list|--no-cipd]"
47  exit 1
48  ;;
49esac
50readonly cipd_ok
51
52case "$(uname)-$(uname -m)" in
53Darwin-x86_64)
54  PLATFORM=mac-amd64
55  ;;
56Linux-x86_64)
57  PLATFORM=linux-amd64
58  ;;
59Linux-aarch64)
60  PLATFORM=linux-arm64
61  ;;
62*)
63  echo 'Unknown operating system.'
64  exit 1
65  ;;
66esac
67readonly PLATFORM
68
69update_stamp() {
70  local -r package="$1" version="$2" download_file="$3"
71  local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp"
72  mkdir -p "$(dirname "$stamp")" && echo "$version" > "$stamp"
73}
74
75verify_stamp() {
76  local verbose=false
77  if [[ "$1" = "--verbose" ]]; then
78    verbose=true
79    shift
80  fi
81  local -r package="$1" version="$2" download_file="$3"
82  local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp"
83  local stamp_version
84  if [[ -r "$stamp" ]]; then
85    stamp_version="$(< "$stamp")"
86  else
87    stamp_version="missing"
88  fi
89  if [[ "$stamp_version" = "$version" ]]; then
90    return 0
91  fi
92  if $verbose; then
93    echo "WARNING: unpacked $package $stamp_version != current $version"
94  fi
95  return 1
96}
97
98list_package() {
99  local -r package="$1" version="$2" download_file="$3" single_file="$4"
100  local -r package_suffix="$5"
101  local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp"
102  local stamp_version
103  if [[ -r "$stamp" ]]; then
104    stamp_version="$(< "$stamp")"
105  else
106    stamp_version="missing"
107  fi
108  local installed="${stamp_version:-missing}"
109  if [[ "$installed" = "$version" ]]; then
110    installed="current"
111  fi
112  echo "$package$package_suffix" "installed=$installed" "current=$version"
113}
114
115verify_file() {
116  local -r file="$1"
117  local -r sum="$2"
118  if [[ "${#sum}" = "40" ]]; then
119    shasum --binary --check --status <<EOF
120$sum *$file
121EOF
122  elif [[ "${#sum}" = "44" ]]; then
123    digest="$(echo $sum | tr '_-' '/+' | base64 --decode | xxd -p -c 64)"
124    shasum -a 256 --binary --check --status <<EOF
125${digest:0:64} *$file
126EOF
127  else
128    echo >&2 "$0: unknown digest type for file ${file}: ${sum}"
129    return 1
130  fi
131}
132
133download_package() {
134  local -r package="$1" version="$2" download_file="$3" single_file="$4"
135  local -r package_suffix="$5"
136  local -r url="${URL_PREFIX}/${package}${package_suffix}/+/${version}"
137  local -r download_file_with_dir="${DOWNLOAD_DIR}/${download_file}"
138
139  # If the stamp file says it's already in place, do nothing more.
140  verify_stamp "$package" "$version" "$download_file" && return
141
142  rm -f -- "$download_file_with_dir"
143  echo "Downloading $url"
144  curl --progress-bar -continue-at=- --location \
145       --create-dirs --output "$download_file_with_dir" "$url" || return
146
147  verify_file "$download_file_with_dir" "$version" || {
148    echo >&2 "*** VERIFICATION ERROR ***"
149    echo >&2 "*** VERIFICATION ERROR *** $download_file from $url"
150    echo >&2 "*** VERIFICATION ERROR *** Not using the file!"
151    return 1
152  }
153
154  echo "Unpacking $download_file"
155  if $single_file; then
156    # The archive contains .cipd* metadata files and a single real file
157    # whose name is the same as the basename of the package.
158    unzip -q -o -d "$DOWNLOAD_DIR" "$download_file_with_dir" \
159          "${package##*/}" || return
160  else
161    local -r dir="${download_file_with_dir%.zip}"
162    rm -rf -- "$dir"
163    unzip -q -d "$dir" "$download_file_with_dir" || return
164  fi
165
166  update_stamp "$package" "$version" "$download_file"
167}
168
169for_each_package() {
170  local package tag version download_file single_file package_suffix status=0
171  local line next=package
172  while read line; do
173    case "$line" in
174    ''|\#*)
175      continue
176      ;;
177    esac
178    eval $next=\$line
179    case $next in
180    package)
181      next=tag
182      continue
183      ;;
184    tag)
185      next=version
186      continue
187      ;;
188    version)
189      next=package
190      ;;
191    esac
192    # Now we've seen all three lines: package, tag, version.
193    case "$package" in
194    fuchsia/*)
195      package="${package#fuchsia/}"
196      ;;
197    *)
198      continue
199      ;;
200    esac
201    case "$package" in
202    tools/*/${PLATFORM})
203      # These are standalone executables contained in a .zip file.
204      package="${package%/*}"
205      download_file="${package#tools/}.zip"
206      single_file=true
207      package_suffix="/${PLATFORM}"
208      ;;
209    */${PLATFORM})
210      package="${package%/*}"
211      # Subdirectories are packed in .zip files.
212      download_file="${package}.zip"
213      single_file=false
214      package_suffix="/${PLATFORM}"
215      ;;
216    firmware/*)
217      # These are the same for every host platform.
218      download_file="${package}.zip"
219      single_file=false
220      package_suffix=''
221      ;;
222    *)
223      # Skip packages for other platforms.
224      continue
225      ;;
226    esac
227    "$@" "$package" "$version" "$download_file" $single_file "$package_suffix" ||
228      status=$?
229  done
230  return $status
231}
232
233find_cipd() {
234  # If the Zircon checkout is part of a jiri checkout that includes
235  # //buildtools, then find cipd there.  Otherwise, if cipd is in
236  # the PATH, take it from there.
237  type -p "${ZIRCON_ROOT}/../buildtools/cipd" || type -p cipd
238}
239
240run_cipd() {
241  local -r CIPD="$1" internal_access="$2" command="$3"
242  shift 3
243  local -r cipd_args=("$@")
244
245  local -a ensure_files=("${PREBUILTS_DIR}/zircon.ensure")
246  if $internal_access; then
247    ensure_files+=("${PREBUILTS_DIR}/zircon_internal.ensure")
248  fi
249
250  # The $ResolvedVersions file name is taken to be relative to the directory
251  # containing the -ensure-file argument.  But since that's - to pipe in the
252  # combined file, make sure CIPD runs with its current directory being the
253  # one where $VERSIONS_FILE is found.
254  (sed '/^\$/!d' "${ensure_files[@]}" && sed '/^\$/d' "${ensure_files[@]}") |
255    (cd "$PREBUILTS_DIR" &&
256     "$CIPD" "$command" -ensure-file - -log-level warning "${cipd_args[@]}")
257  rc=$?
258  if [[ $rc -ne 0 ]]; then
259    echo >&2 "$0: $CIPD failed.  For direct downloads, remove cipd from PATH."
260    return $rc
261  fi
262}
263
264update_via_cipd() {
265  run_cipd "$1" "$2" ensure -root "$DOWNLOAD_DIR"
266
267  # Now update the .stamp files so that --verify will be happy.
268  for_each_package update_stamp < "$VERSIONS_FILE"
269}
270
271write_sysroot_path() {
272  local -r package="$1" version="$2"
273  if [[ "$package" = sysroot ]]; then
274    echo "SYSROOT_${PLATFORM}_PATH = \$(LKMAKEROOT)/prebuilt/downloads/sysroot"
275  fi
276}
277
278write_config_mk() {
279  local -r internal_access="$1"
280  local -r config_mk="${PREBUILTS_DIR}/config.mk"
281  rm -f -- "$config_mk"
282  echo > "$config_mk" "# Generated by $0.  DO NOT EDIT!"'
283
284PREBUILT_CHECK := $(shell $(LKMAKEROOT)/scripts/download-prebuilt --verify)
285ifneq (,$(strip $(PREBUILT_CHECK)))
286$(warning)
287$(warning $(PREBUILT_CHECK))
288$(warning run scripts/download-prebuilt)
289$(warning)
290endif
291
292ARCH_x86_64_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/gcc/bin/x86_64-elf-
293ARCH_arm64_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/gcc/bin/aarch64-elf-
294CLANG_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/clang/bin/'
295
296  for_each_package write_sysroot_path < "$VERSIONS_FILE" >> "$config_mk"
297
298  echo >> "$config_mk" "INTERNAL_ACCESS := ${internal_access}"
299}
300
301update() {
302  local CIPD
303  local internal_access=false
304  if $cipd_ok && CIPD="$(find_cipd)"; then
305    # We have CIPD, so use it.
306    if [[ "$("$CIPD" ls fuchsia_internal)" != "No matching packages." ]]; then
307      internal_access=true
308    fi
309    update_via_cipd "$CIPD" "$internal_access" || return
310  else
311    # We don't have CIPD, so don't use it.
312    for_each_package download_package < "$VERSIONS_FILE" || return
313  fi
314  write_config_mk "$internal_access"
315}
316
317verify() {
318  for_each_package verify_stamp --verbose < "$VERSIONS_FILE"
319}
320
321list() {
322  for_each_package list_package < "$VERSIONS_FILE"
323}
324
325resolve() {
326  run_cipd "$(find_cipd)" true ensure-file-resolve
327}
328
329$mode
330