From 2a0cbda8bcad4aa14aa1f679fdfe84e7889b3785 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Fri, 7 Aug 2020 16:22:05 -0700 Subject: [PATCH] devops: mirror chromium builds to our CDN (#3357) Start mirroring chromium builds to our CDN. References #3259 --- .../buildbots/buildbot-ubuntu-18.04.sh | 6 ++++ .../checkout_build_archive_upload.sh | 26 +++++++++++++- browser_patches/chromium/.gitignore | 1 + browser_patches/chromium/BUILD_NUMBER | 1 + browser_patches/chromium/archive.sh | 34 +++++++++++++++++++ browser_patches/chromium/build.sh | 33 ++++++++++++++++++ browser_patches/chromium/clean.sh | 9 +++++ browser_patches/prepare_checkout.sh | 5 ++- browser_patches/tools/check_cdn.sh | 18 ++++++++++ 9 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 browser_patches/chromium/.gitignore create mode 100644 browser_patches/chromium/BUILD_NUMBER create mode 100755 browser_patches/chromium/archive.sh create mode 100755 browser_patches/chromium/build.sh create mode 100755 browser_patches/chromium/clean.sh diff --git a/browser_patches/buildbots/buildbot-ubuntu-18.04.sh b/browser_patches/buildbots/buildbot-ubuntu-18.04.sh index ccb264b5f4..8449bcd364 100755 --- a/browser_patches/buildbots/buildbot-ubuntu-18.04.sh +++ b/browser_patches/buildbots/buildbot-ubuntu-18.04.sh @@ -67,3 +67,9 @@ git pull origin master git pull origin master ../checkout_build_archive_upload.sh webkit-ubuntu-18.04 >/tmp/$(basename $0)--webkit.log || true + +git pull origin master +../checkout_build_archive_upload.sh chromium-linux-mirror-to-cdn >/tmp/$(basename $0)--chromium-linux.log || true +../checkout_build_archive_upload.sh chromium-mac-mirror-to-cdn >/tmp/$(basename $0)--chromium-mac.log || true +../checkout_build_archive_upload.sh chromium-win32-mirror-to-cdn >/tmp/$(basename $0)--chromium-win32.log || true +../checkout_build_archive_upload.sh chromium-win64-mirror-to-cdn >/tmp/$(basename $0)--chromium-win64.log || true diff --git a/browser_patches/checkout_build_archive_upload.sh b/browser_patches/checkout_build_archive_upload.sh index 6968176692..8659a03cb0 100755 --- a/browser_patches/checkout_build_archive_upload.sh +++ b/browser_patches/checkout_build_archive_upload.sh @@ -36,7 +36,31 @@ BUILD_FLAVOR="$1" BUILD_BLOB_NAME="" EXPECTED_HOST_OS="" EXPECTED_HOST_OS_VERSION="" -if [[ "$BUILD_FLAVOR" == "firefox-ubuntu-18.04" ]]; then +if [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then + BROWSER_NAME="chromium" + EXTRA_BUILD_ARGS="--linux" + EXPECTED_HOST_OS="Ubuntu" + EXPECTED_HOST_OS_VERSION="18.04" + BUILD_BLOB_NAME="chromium-linux.zip" +elif [[ "$BUILD_FLAVOR" == "chromium-mac-mirror-to-cdn" ]]; then + BROWSER_NAME="chromium" + EXTRA_BUILD_ARGS="--mac" + EXPECTED_HOST_OS="Ubuntu" + EXPECTED_HOST_OS_VERSION="18.04" + BUILD_BLOB_NAME="chromium-mac.zip" +elif [[ "$BUILD_FLAVOR" == "chromium-win32-mirror-to-cdn" ]]; then + BROWSER_NAME="chromium" + EXTRA_BUILD_ARGS="--win32" + EXPECTED_HOST_OS="Ubuntu" + EXPECTED_HOST_OS_VERSION="18.04" + BUILD_BLOB_NAME="chromium-win32.zip" +elif [[ "$BUILD_FLAVOR" == "chromium-win64-mirror-to-cdn" ]]; then + BROWSER_NAME="chromium" + EXTRA_BUILD_ARGS="--win64" + EXPECTED_HOST_OS="Ubuntu" + EXPECTED_HOST_OS_VERSION="18.04" + BUILD_BLOB_NAME="chromium-win64.zip" +elif [[ "$BUILD_FLAVOR" == "firefox-ubuntu-18.04" ]]; then BROWSER_NAME="firefox" EXPECTED_HOST_OS="Ubuntu" EXPECTED_HOST_OS_VERSION="18.04" diff --git a/browser_patches/chromium/.gitignore b/browser_patches/chromium/.gitignore new file mode 100644 index 0000000000..1f57b975cd --- /dev/null +++ b/browser_patches/chromium/.gitignore @@ -0,0 +1 @@ +/output diff --git a/browser_patches/chromium/BUILD_NUMBER b/browser_patches/chromium/BUILD_NUMBER new file mode 100644 index 0000000000..5c111e426a --- /dev/null +++ b/browser_patches/chromium/BUILD_NUMBER @@ -0,0 +1 @@ +792639 diff --git a/browser_patches/chromium/archive.sh b/browser_patches/chromium/archive.sh new file mode 100755 index 0000000000..409bf9152a --- /dev/null +++ b/browser_patches/chromium/archive.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e +set +x + +if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then + echo "usage: $(basename $0) [output-absolute-path]" + echo + echo "Generate distributable .zip archive from ./output folder that was previously downloaded." + echo + exit 0 +fi + +ZIP_PATH=$1 +if [[ $ZIP_PATH != /* ]]; then + echo "ERROR: path $ZIP_PATH is not absolute" + exit 1 +fi +if [[ $ZIP_PATH != *.zip ]]; then + echo "ERROR: path $ZIP_PATH must have .zip extension" + exit 1 +fi +if [[ -f $ZIP_PATH ]]; then + echo "ERROR: path $ZIP_PATH exists; can't do anything." + exit 1 +fi +if ! [[ -d $(dirname $ZIP_PATH) ]]; then + echo "ERROR: folder for path $($ZIP_PATH) does not exist." + exit 1 +fi + +trap "cd $(pwd -P)" EXIT +cd "$(dirname $0)" + +cp output/archive.zip $ZIP_PATH diff --git a/browser_patches/chromium/build.sh b/browser_patches/chromium/build.sh new file mode 100755 index 0000000000..3fe6e247fe --- /dev/null +++ b/browser_patches/chromium/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e +set +x + +trap "cd $(pwd -P)" EXIT +cd "$(dirname $0)" + +mkdir -p output +cd output + +BUILD_NUMBER=$(head -1 ../BUILD_NUMBER) +FOLDER_NAME="" +ZIP_NAME="" +if [[ $1 == "--win32" ]]; then + FOLDER_NAME="Win" + ZIP_NAME="chrome-win.zip" +elif [[ $1 == "--win64" ]]; then + FOLDER_NAME="Win_x64" + ZIP_NAME="chrome-win.zip" +elif [[ $1 == "--mac" ]]; then + FOLDER_NAME="Mac" + ZIP_NAME="chrome-mac.zip" +elif [[ $1 == "--linux" ]]; then + FOLDER_NAME="Linux_x64" + ZIP_NAME="chrome-linux.zip" +else + echo "ERROR: unknown platform to build: $1" + exit 1 +fi + +URL="https://storage.googleapis.com/chromium-browser-snapshots/${FOLDER_NAME}/${BUILD_NUMBER}/${ZIP_NAME}" +curl --output archive.zip "${URL}" + diff --git a/browser_patches/chromium/clean.sh b/browser_patches/chromium/clean.sh new file mode 100755 index 0000000000..db4d36c08e --- /dev/null +++ b/browser_patches/chromium/clean.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e +set +x + +trap "cd $(pwd -P)" EXIT +cd "$(dirname $0)" + +rm -rf output + diff --git a/browser_patches/prepare_checkout.sh b/browser_patches/prepare_checkout.sh index f5e42c7af7..0686d7e74f 100755 --- a/browser_patches/prepare_checkout.sh +++ b/browser_patches/prepare_checkout.sh @@ -33,7 +33,10 @@ PATCHES_PATH="" BUILD_NUMBER="" WEBKIT_EXTRA_FOLDER_PATH="" FIREFOX_EXTRA_FOLDER_PATH="" -if [[ ("$1" == "firefox") || ("$1" == "firefox/") || ("$1" == "ff") ]]; then +if [[ ("$1" == "chromium") || ("$1" == "chromium/") || ("$1" == "cr") ]]; then + echo "FYI: chromium checkout is not supported." + exit 0 +elif [[ ("$1" == "firefox") || ("$1" == "firefox/") || ("$1" == "ff") ]]; then FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout"; CHECKOUT_PATH="$PWD/firefox/checkout" PATCHES_PATH="$PWD/firefox/patches" diff --git a/browser_patches/tools/check_cdn.sh b/browser_patches/tools/check_cdn.sh index bb6e9c4bfe..bbe39f9a83 100755 --- a/browser_patches/tools/check_cdn.sh +++ b/browser_patches/tools/check_cdn.sh @@ -51,6 +51,20 @@ WK_ALIASES=( "WK-WIN64" ) +CR_REVISION=$(head -1 ../chromium/BUILD_NUMBER) +CR_ARCHIVES=( + "$HOST/chromium/%s/chromium-mac.zip" + "$HOST/chromium/%s/chromium-linux.zip" + "$HOST/chromium/%s/chromium-win32.zip" + "$HOST/chromium/%s/chromium-win64.zip" +) +CR_ALIASES=( + "CR-MAC" + "CR-LINUX" + "CR-WIN32" + "CR-WIN64" +) + COLUMN="%-18s" # COLORS RED=$'\e[1;31m' @@ -69,6 +83,10 @@ elif [[ ("$1" == "webkit") || ("$1" == "webkit/") ]]; then REVISION=$WK_REVISION ARCHIVES=("${WK_ARCHIVES[@]}") ALIASES=("${WK_ALIASES[@]}") +elif [[ ("$1" == "chromium") || ("$1" == "chromium/") ]]; then + REVISION=$CR_REVISION + ARCHIVES=("${CR_ARCHIVES[@]}") + ALIASES=("${CR_ALIASES[@]}") else echo ERROR: unknown browser - "$1" exit 1