#!/usr/bin/env bash # Copyright 2018 The Fuchsia Authors # # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file or at # https://opensource.org/licenses/MIT readonly SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" readonly ZIRCON_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" readonly PREBUILTS_DIR="$(cd "${ZIRCON_ROOT}/prebuilt" && pwd)" readonly DOWNLOAD_DIR="${PREBUILTS_DIR}/downloads" readonly ENSURE_FILE="${PREBUILTS_DIR}/zircon.ensure" readonly VERSIONS_FILE="${PREBUILTS_DIR}/zircon.versions" readonly URL_PREFIX="https://chrome-infra-packages.appspot.com/dl/fuchsia" # This script assumes that ENSURE_FILE and VERSIONS_FILE match up. # `cipd ensure` checks this and `cips ensure-file-resolve` ensures it. # When a `cipd` binary is available, ENSURE_FILE controls the downloads # and points `cipd` at VERSIONS_FILE for resolved pinned versions. # Otherwise, VERSIONS_FILE controls the downloads directly. In both # cases we write $DOWNLOAD_DIR/$PACKAGE.stamp files with the versions # from VERSIONS_FILE on faith that that's what we just unpacked. set -o pipefail cipd_ok=true case "$#:$1" in 0:) mode=update ;; 1:--verify) mode=verify ;; 1:--list) mode=list ;; 1:--resolve) mode=resolve ;; 1:--no-cipd) mode=update cipd_ok=false ;; *) echo >&2 "Usage: $0 [--verify|--list|--no-cipd]" exit 1 ;; esac readonly cipd_ok case "$(uname)-$(uname -m)" in Darwin-x86_64) PLATFORM=mac-amd64 ;; Linux-x86_64) PLATFORM=linux-amd64 ;; Linux-aarch64) PLATFORM=linux-arm64 ;; *) echo 'Unknown operating system.' exit 1 ;; esac readonly PLATFORM update_stamp() { local -r package="$1" version="$2" download_file="$3" local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp" mkdir -p "$(dirname "$stamp")" && echo "$version" > "$stamp" } verify_stamp() { local verbose=false if [[ "$1" = "--verbose" ]]; then verbose=true shift fi local -r package="$1" version="$2" download_file="$3" local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp" local stamp_version if [[ -r "$stamp" ]]; then stamp_version="$(< "$stamp")" else stamp_version="missing" fi if [[ "$stamp_version" = "$version" ]]; then return 0 fi if $verbose; then echo "WARNING: unpacked $package $stamp_version != current $version" fi return 1 } list_package() { local -r package="$1" version="$2" download_file="$3" single_file="$4" local -r package_suffix="$5" local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp" local stamp_version if [[ -r "$stamp" ]]; then stamp_version="$(< "$stamp")" else stamp_version="missing" fi local installed="${stamp_version:-missing}" if [[ "$installed" = "$version" ]]; then installed="current" fi echo "$package$package_suffix" "installed=$installed" "current=$version" } verify_file() { local -r file="$1" local -r sum="$2" if [[ "${#sum}" = "40" ]]; then shasum --binary --check --status <&2 "$0: unknown digest type for file ${file}: ${sum}" return 1 fi } download_package() { local -r package="$1" version="$2" download_file="$3" single_file="$4" local -r package_suffix="$5" local -r url="${URL_PREFIX}/${package}${package_suffix}/+/${version}" local -r download_file_with_dir="${DOWNLOAD_DIR}/${download_file}" # If the stamp file says it's already in place, do nothing more. verify_stamp "$package" "$version" "$download_file" && return rm -f -- "$download_file_with_dir" echo "Downloading $url" curl --progress-bar -continue-at=- --location \ --create-dirs --output "$download_file_with_dir" "$url" || return verify_file "$download_file_with_dir" "$version" || { echo >&2 "*** VERIFICATION ERROR ***" echo >&2 "*** VERIFICATION ERROR *** $download_file from $url" echo >&2 "*** VERIFICATION ERROR *** Not using the file!" return 1 } echo "Unpacking $download_file" if $single_file; then # The archive contains .cipd* metadata files and a single real file # whose name is the same as the basename of the package. unzip -q -o -d "$DOWNLOAD_DIR" "$download_file_with_dir" \ "${package##*/}" || return else local -r dir="${download_file_with_dir%.zip}" rm -rf -- "$dir" unzip -q -d "$dir" "$download_file_with_dir" || return fi update_stamp "$package" "$version" "$download_file" } for_each_package() { local package tag version download_file single_file package_suffix status=0 local line next=package while read line; do case "$line" in ''|\#*) continue ;; esac eval $next=\$line case $next in package) next=tag continue ;; tag) next=version continue ;; version) next=package ;; esac # Now we've seen all three lines: package, tag, version. case "$package" in fuchsia/*) package="${package#fuchsia/}" ;; *) continue ;; esac case "$package" in tools/*/${PLATFORM}) # These are standalone executables contained in a .zip file. package="${package%/*}" download_file="${package#tools/}.zip" single_file=true package_suffix="/${PLATFORM}" ;; */${PLATFORM}) package="${package%/*}" # Subdirectories are packed in .zip files. download_file="${package}.zip" single_file=false package_suffix="/${PLATFORM}" ;; firmware/*) # These are the same for every host platform. download_file="${package}.zip" single_file=false package_suffix='' ;; *) # Skip packages for other platforms. continue ;; esac "$@" "$package" "$version" "$download_file" $single_file "$package_suffix" || status=$? done return $status } find_cipd() { # If the Zircon checkout is part of a jiri checkout that includes # //buildtools, then find cipd there. Otherwise, if cipd is in # the PATH, take it from there. type -p "${ZIRCON_ROOT}/../buildtools/cipd" || type -p cipd } run_cipd() { local -r CIPD="$1" internal_access="$2" command="$3" shift 3 local -r cipd_args=("$@") local -a ensure_files=("${PREBUILTS_DIR}/zircon.ensure") if $internal_access; then ensure_files+=("${PREBUILTS_DIR}/zircon_internal.ensure") fi # The $ResolvedVersions file name is taken to be relative to the directory # containing the -ensure-file argument. But since that's - to pipe in the # combined file, make sure CIPD runs with its current directory being the # one where $VERSIONS_FILE is found. (sed '/^\$/!d' "${ensure_files[@]}" && sed '/^\$/d' "${ensure_files[@]}") | (cd "$PREBUILTS_DIR" && "$CIPD" "$command" -ensure-file - -log-level warning "${cipd_args[@]}") rc=$? if [[ $rc -ne 0 ]]; then echo >&2 "$0: $CIPD failed. For direct downloads, remove cipd from PATH." return $rc fi } update_via_cipd() { run_cipd "$1" "$2" ensure -root "$DOWNLOAD_DIR" # Now update the .stamp files so that --verify will be happy. for_each_package update_stamp < "$VERSIONS_FILE" } write_sysroot_path() { local -r package="$1" version="$2" if [[ "$package" = sysroot ]]; then echo "SYSROOT_${PLATFORM}_PATH = \$(LKMAKEROOT)/prebuilt/downloads/sysroot" fi } write_config_mk() { local -r internal_access="$1" local -r config_mk="${PREBUILTS_DIR}/config.mk" rm -f -- "$config_mk" echo > "$config_mk" "# Generated by $0. DO NOT EDIT!"' PREBUILT_CHECK := $(shell $(LKMAKEROOT)/scripts/download-prebuilt --verify) ifneq (,$(strip $(PREBUILT_CHECK))) $(warning) $(warning $(PREBUILT_CHECK)) $(warning run scripts/download-prebuilt) $(warning) endif ARCH_x86_64_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/gcc/bin/x86_64-elf- ARCH_arm64_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/gcc/bin/aarch64-elf- CLANG_TOOLCHAIN_PREFIX = $(LKMAKEROOT)/prebuilt/downloads/clang/bin/' for_each_package write_sysroot_path < "$VERSIONS_FILE" >> "$config_mk" echo >> "$config_mk" "INTERNAL_ACCESS := ${internal_access}" } update() { local CIPD local internal_access=false if $cipd_ok && CIPD="$(find_cipd)"; then # We have CIPD, so use it. if [[ "$("$CIPD" ls fuchsia_internal)" != "No matching packages." ]]; then internal_access=true fi update_via_cipd "$CIPD" "$internal_access" || return else # We don't have CIPD, so don't use it. for_each_package download_package < "$VERSIONS_FILE" || return fi write_config_mk "$internal_access" } verify() { for_each_package verify_stamp --verbose < "$VERSIONS_FILE" } list() { for_each_package list_package < "$VERSIONS_FILE" } resolve() { run_cipd "$(find_cipd)" true ensure-file-resolve } $mode