From a80fa0869893eeae171ce9b4cc728e4564c8bb6b Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Thu, 14 Oct 2021 10:20:06 -0700 Subject: [PATCH] devops: move Firefox and WebKit checkouts to $HOME (#9485) This moves default Firefox and WebKit checkouts to $HOME folder, unless browser specific env variables are specified. In other words: - Firefox checkouts goes to `$HOME/firefox` unless there's a `$FF_CHECKOUT_PATH` that specifies a custom location. - WebKit checkout goes to `$HOME/webkit` unless there's a `$WK_CHECKOUT_PATH` that specifies a custom location. With this change, all build bots will now re-use checkouts between builds, which should speed up compilation. --- browser_patches/export.sh | 42 ++++++++++-------- browser_patches/firefox-beta/archive.sh | 4 +- browser_patches/firefox-beta/build.sh | 2 +- browser_patches/firefox-beta/clean.sh | 2 +- browser_patches/firefox/archive.sh | 6 +-- browser_patches/firefox/build.sh | 2 +- browser_patches/firefox/clean.sh | 3 +- browser_patches/prepare_checkout.sh | 54 +++++++++++------------ browser_patches/webkit/archive.sh | 2 +- browser_patches/webkit/build.sh | 2 +- browser_patches/webkit/clean.sh | 4 +- browser_patches/webkit/concat_protocol.js | 2 +- browser_patches/webkit/pw_run.sh | 14 +++--- 13 files changed, 70 insertions(+), 69 deletions(-) diff --git a/browser_patches/export.sh b/browser_patches/export.sh index dec6c8da11..209e4a32b7 100755 --- a/browser_patches/export.sh +++ b/browser_patches/export.sh @@ -41,46 +41,50 @@ EXPORT_PATH="" EXTRA_FOLDER_PW_PATH="" EXTRA_FOLDER_CHECKOUT_RELPATH="" if [[ ("$1" == "firefox") || ("$1" == "firefox/") || ("$1" == "ff") ]]; then - FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout"; - CHECKOUT_PATH="$PWD/firefox/checkout" + if [[ -z "${FF_CHECKOUT_PATH}" ]]; then + FRIENDLY_CHECKOUT_PATH='$HOME/firefox'; + CHECKOUT_PATH="$HOME/firefox" + else + echo "WARNING: using checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" + CHECKOUT_PATH="${FF_CHECKOUT_PATH}" + FRIENDLY_CHECKOUT_PATH="" + fi + EXTRA_FOLDER_PW_PATH="$PWD/firefox/juggler" EXTRA_FOLDER_CHECKOUT_RELPATH="juggler" EXPORT_PATH="$PWD/firefox" BUILD_NUMBER_UPSTREAM_URL="https://raw.githubusercontent.com/microsoft/playwright/master/browser_patches/firefox/BUILD_NUMBER" source "./firefox/UPSTREAM_CONFIG.sh" - if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then +elif [[ ("$1" == "firefox-beta") || ("$1" == "ff-beta") ]]; then + if [[ -z "${FF_CHECKOUT_PATH}" ]]; then + FRIENDLY_CHECKOUT_PATH='$HOME/firefox'; + CHECKOUT_PATH="$HOME/firefox" + else echo "WARNING: using checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" CHECKOUT_PATH="${FF_CHECKOUT_PATH}" FRIENDLY_CHECKOUT_PATH="" fi -elif [[ ("$1" == "firefox-beta") || ("$1" == "ff-beta") ]]; then - # NOTE: firefox-beta re-uses firefox checkout. - FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout"; - CHECKOUT_PATH="$PWD/firefox/checkout" EXTRA_FOLDER_PW_PATH="$PWD/firefox-beta/juggler" EXTRA_FOLDER_CHECKOUT_RELPATH="juggler" EXPORT_PATH="$PWD/firefox-beta" BUILD_NUMBER_UPSTREAM_URL="https://raw.githubusercontent.com/microsoft/playwright/master/browser_patches/firefox-beta/BUILD_NUMBER" source "./firefox-beta/UPSTREAM_CONFIG.sh" - if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then - echo "WARNING: using checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" - CHECKOUT_PATH="${FF_CHECKOUT_PATH}" - FRIENDLY_CHECKOUT_PATH="" - fi elif [[ ("$1" == "webkit") || ("$1" == "webkit/") || ("$1" == "wk") ]]; then - FRIENDLY_CHECKOUT_PATH="//browser_patches/webkit/checkout"; - CHECKOUT_PATH="$PWD/webkit/checkout" + if [[ -z "${WK_CHECKOUT_PATH}" ]]; then + FRIENDLY_CHECKOUT_PATH='$HOME/webkit'; + CHECKOUT_PATH="$HOME/webkit" + else + echo "WARNING: using checkout path from WK_CHECKOUT_PATH env: ${WK_CHECKOUT_PATH}" + CHECKOUT_PATH="${WK_CHECKOUT_PATH}" + FRIENDLY_CHECKOUT_PATH="" + fi + EXTRA_FOLDER_PW_PATH="$PWD/webkit/embedder/Playwright" EXTRA_FOLDER_CHECKOUT_RELPATH="Tools/Playwright" EXPORT_PATH="$PWD/webkit" BUILD_NUMBER_UPSTREAM_URL="https://raw.githubusercontent.com/microsoft/playwright/master/browser_patches/webkit/BUILD_NUMBER" source "./webkit/UPSTREAM_CONFIG.sh" - if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then - echo "WARNING: using checkout path from WK_CHECKOUT_PATH env: ${WK_CHECKOUT_PATH}" - CHECKOUT_PATH="${WK_CHECKOUT_PATH}" - FRIENDLY_CHECKOUT_PATH="" - fi else echo ERROR: unknown browser to export - "$1" exit 1 diff --git a/browser_patches/firefox-beta/archive.sh b/browser_patches/firefox-beta/archive.sh index c23d782e0c..91efbc4269 100755 --- a/browser_patches/firefox-beta/archive.sh +++ b/browser_patches/firefox-beta/archive.sh @@ -5,7 +5,7 @@ set +x if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then echo "usage: $(basename "$0") [output-absolute-path]" echo - echo "Generate distributable .zip archive from ./checkout folder that was previously built." + echo "Generate distributable .zip archive from Firefox checkout folder that was previously built." echo exit 0 fi @@ -36,7 +36,7 @@ if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then cd "${FF_CHECKOUT_PATH}" echo "WARNING: checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" else - cd "../firefox/checkout" + cd "$HOME/firefox" fi OBJ_FOLDER="obj-build-playwright" diff --git a/browser_patches/firefox-beta/build.sh b/browser_patches/firefox-beta/build.sh index b9416097df..d69db02c42 100755 --- a/browser_patches/firefox-beta/build.sh +++ b/browser_patches/firefox-beta/build.sh @@ -15,7 +15,7 @@ if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then cd "${FF_CHECKOUT_PATH}" echo "WARNING: checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" else - cd "../firefox/checkout" + cd "$HOME/firefox" fi rm -rf .mozconfig diff --git a/browser_patches/firefox-beta/clean.sh b/browser_patches/firefox-beta/clean.sh index ef80f7ea07..6dbb181ac9 100755 --- a/browser_patches/firefox-beta/clean.sh +++ b/browser_patches/firefox-beta/clean.sh @@ -8,7 +8,7 @@ if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then echo "WARNING: checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" else cd "$(dirname "$0")" - cd "../firefox/checkout" + cd "$HOME/firefox" fi OBJ_FOLDER="obj-build-playwright" diff --git a/browser_patches/firefox/archive.sh b/browser_patches/firefox/archive.sh index a3bc22ddd6..ec364beeba 100755 --- a/browser_patches/firefox/archive.sh +++ b/browser_patches/firefox/archive.sh @@ -5,7 +5,7 @@ set +x if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then echo "usage: $(basename "$0") [output-absolute-path]" echo - echo "Generate distributable .zip archive from ./checkout folder that was previously built." + echo "Generate distributable .zip archive from Firefox checkout folder that was previously built." echo exit 0 fi @@ -36,7 +36,7 @@ if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then cd "${FF_CHECKOUT_PATH}" echo "WARNING: checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" else - cd "checkout" + cd "$HOME/firefox" fi OBJ_FOLDER="obj-build-playwright" @@ -45,7 +45,7 @@ OBJ_FOLDER="obj-build-playwright" node "${SCRIPT_FOLDER}"/install-preferences.js "$PWD"/$OBJ_FOLDER/dist/firefox if ! [[ -d $OBJ_FOLDER/dist/firefox ]]; then - echo "ERROR: cannot find $OBJ_FOLDER/dist/firefox folder in the checkout/. Did you build?" + echo "ERROR: cannot find $OBJ_FOLDER/dist/firefox folder in the firefox checkout. Did you build?" exit 1; fi diff --git a/browser_patches/firefox/build.sh b/browser_patches/firefox/build.sh index a9eb07e827..63915e1b4c 100755 --- a/browser_patches/firefox/build.sh +++ b/browser_patches/firefox/build.sh @@ -19,7 +19,7 @@ if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then cd "${FF_CHECKOUT_PATH}" echo "WARNING: checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" else - cd "../firefox/checkout" + cd "$HOME/firefox" fi rm -rf .mozconfig diff --git a/browser_patches/firefox/clean.sh b/browser_patches/firefox/clean.sh index a849343d30..907d7ceec9 100755 --- a/browser_patches/firefox/clean.sh +++ b/browser_patches/firefox/clean.sh @@ -7,8 +7,7 @@ if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then cd "${FF_CHECKOUT_PATH}" echo "WARNING: checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" else - cd "$(dirname "$0")" - cd "checkout" + cd "$HOME/firefox" fi OBJ_FOLDER="obj-build-playwright" diff --git a/browser_patches/prepare_checkout.sh b/browser_patches/prepare_checkout.sh index 9685a05c28..34fdf87b4b 100755 --- a/browser_patches/prepare_checkout.sh +++ b/browser_patches/prepare_checkout.sh @@ -82,55 +82,53 @@ elif [[ ("$1" == "winldd") || ("$1" == "winldd/") ]]; then echo "FYI: winldd source code is available right away" exit 0 elif [[ ("$1" == "firefox") || ("$1" == "firefox/") || ("$1" == "ff") ]]; then - FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout"; - CHECKOUT_PATH="$PWD/firefox/checkout" - PATCHES_PATH="$PWD/firefox/patches" - FIREFOX_EXTRA_FOLDER_PATH="$PWD/firefox/juggler" - BUILD_NUMBER=$(head -1 "$PWD/firefox/BUILD_NUMBER") - source "./firefox/UPSTREAM_CONFIG.sh" - if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then + if [[ -z "${FF_CHECKOUT_PATH}" ]]; then + FRIENDLY_CHECKOUT_PATH='$HOME/firefox'; + CHECKOUT_PATH="$HOME/firefox" + else echo "WARNING: using checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" CHECKOUT_PATH="${FF_CHECKOUT_PATH}" FRIENDLY_CHECKOUT_PATH="" fi + + PATCHES_PATH="$PWD/firefox/patches" + FIREFOX_EXTRA_FOLDER_PATH="$PWD/firefox/juggler" + BUILD_NUMBER=$(head -1 "$PWD/firefox/BUILD_NUMBER") + source "./firefox/UPSTREAM_CONFIG.sh" elif [[ ("$1" == "firefox-beta") || ("$1" == "ff-beta") ]]; then # NOTE: firefox-beta re-uses firefox checkout. - FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout"; - CHECKOUT_PATH="$PWD/firefox/checkout" + if [[ -z "${FF_CHECKOUT_PATH}" ]]; then + FRIENDLY_CHECKOUT_PATH='$HOME/firefox'; + CHECKOUT_PATH="$HOME/firefox" + else + echo "WARNING: using checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" + CHECKOUT_PATH="${FF_CHECKOUT_PATH}" + FRIENDLY_CHECKOUT_PATH="" + fi PATCHES_PATH="$PWD/firefox-beta/patches" FIREFOX_EXTRA_FOLDER_PATH="$PWD/firefox-beta/juggler" BUILD_NUMBER=$(head -1 "$PWD/firefox-beta/BUILD_NUMBER") source "./firefox-beta/UPSTREAM_CONFIG.sh" - if [[ ! -z "${FF_CHECKOUT_PATH}" ]]; then - echo "WARNING: using checkout path from FF_CHECKOUT_PATH env: ${FF_CHECKOUT_PATH}" - CHECKOUT_PATH="${FF_CHECKOUT_PATH}" - FRIENDLY_CHECKOUT_PATH="" - fi elif [[ ("$1" == "webkit") || ("$1" == "webkit/") || ("$1" == "wk") ]]; then - FRIENDLY_CHECKOUT_PATH="//browser_patches/webkit/checkout"; - CHECKOUT_PATH="$PWD/webkit/checkout" - PATCHES_PATH="$PWD/webkit/patches" - WEBKIT_EXTRA_FOLDER_PATH="$PWD/webkit/embedder/Playwright" - BUILD_NUMBER=$(head -1 "$PWD/webkit/BUILD_NUMBER") - source "./webkit/UPSTREAM_CONFIG.sh" - if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then + if [[ -z "${WK_CHECKOUT_PATH}" ]]; then + FRIENDLY_CHECKOUT_PATH='$HOME/webkit'; + CHECKOUT_PATH="$HOME/webkit" + else echo "WARNING: using checkout path from WK_CHECKOUT_PATH env: ${WK_CHECKOUT_PATH}" CHECKOUT_PATH="${WK_CHECKOUT_PATH}" FRIENDLY_CHECKOUT_PATH="" fi + + PATCHES_PATH="$PWD/webkit/patches" + WEBKIT_EXTRA_FOLDER_PATH="$PWD/webkit/embedder/Playwright" + BUILD_NUMBER=$(head -1 "$PWD/webkit/BUILD_NUMBER") + source "./webkit/UPSTREAM_CONFIG.sh" else echo ERROR: unknown browser - "$1" exit 1 fi -# we will use this just for beauty. -if [[ $# == 2 ]]; then - echo "WARNING: using custom checkout path $CHECKOUT_PATH" - CHECKOUT_PATH=$2 - FRIENDLY_CHECKOUT_PATH="" -fi - # if there's no checkout folder - checkout one. if ! [[ -d $CHECKOUT_PATH ]]; then echo "-- $FRIENDLY_CHECKOUT_PATH is missing - checking out.." diff --git a/browser_patches/webkit/archive.sh b/browser_patches/webkit/archive.sh index d2f5782cbb..93e22cb052 100755 --- a/browser_patches/webkit/archive.sh +++ b/browser_patches/webkit/archive.sh @@ -33,7 +33,7 @@ main() { cd "${WK_CHECKOUT_PATH}" echo "WARNING: checkout path from WK_CHECKOUT_PATH env: ${WK_CHECKOUT_PATH}" else - cd "checkout" + cd "$HOME/webkit" fi set -x diff --git a/browser_patches/webkit/build.sh b/browser_patches/webkit/build.sh index 1bcac55562..392c759b6d 100755 --- a/browser_patches/webkit/build.sh +++ b/browser_patches/webkit/build.sh @@ -40,7 +40,7 @@ if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then cd "${WK_CHECKOUT_PATH}" echo "WARNING: checkout path from WK_CHECKOUT_PATH env: ${WK_CHECKOUT_PATH}" else - cd "checkout" + cd "$HOME/webkit" fi if [[ "$(uname)" == "Darwin" ]]; then diff --git a/browser_patches/webkit/clean.sh b/browser_patches/webkit/clean.sh index df9e0467f3..0a843871a7 100755 --- a/browser_patches/webkit/clean.sh +++ b/browser_patches/webkit/clean.sh @@ -9,7 +9,7 @@ if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then cd "${WK_CHECKOUT_PATH}" echo "WARNING: checkout path from WK_CHECKOUT_PATH env: ${WK_CHECKOUT_PATH}" else - cd "checkout" + cd "$HOME/webkit" fi if [[ "$(uname)" == "Darwin" ]]; then @@ -24,4 +24,4 @@ else if [[ -d ./WebKitBuild/WPE ]]; then rm -rf ./WebKitBuild/WPE/Release fi -fi \ No newline at end of file +fi diff --git a/browser_patches/webkit/concat_protocol.js b/browser_patches/webkit/concat_protocol.js index 9d2d405d47..3e28fb6bb0 100644 --- a/browser_patches/webkit/concat_protocol.js +++ b/browser_patches/webkit/concat_protocol.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const checkoutPath = process.env.WK_CHECKOUT_PATH || path.join(__dirname, 'checkout'); +const checkoutPath = process.env.WK_CHECKOUT_PATH || path.join(process.env.HOME, 'webkit'); const protocolDir = path.join(checkoutPath, './Source/JavaScriptCore/inspector/protocol'); const files = fs.readdirSync(protocolDir).filter(f => f.endsWith('.json')).map(f => path.join(protocolDir, f)); const json = files.map(file => JSON.parse(fs.readFileSync(file))); diff --git a/browser_patches/webkit/pw_run.sh b/browser_patches/webkit/pw_run.sh index 10c166f234..d14466490a 100755 --- a/browser_patches/webkit/pw_run.sh +++ b/browser_patches/webkit/pw_run.sh @@ -4,8 +4,8 @@ function runOSX() { # if script is run as-is if [[ -f "${SCRIPT_PATH}/EXPECTED_BUILDS" && -n "$WK_CHECKOUT_PATH" && -d "$WK_CHECKOUT_PATH/WebKitBuild/Release/Playwright.app" ]]; then DYLIB_PATH="$WK_CHECKOUT_PATH/WebKitBuild/Release" - elif [[ -f "${SCRIPT_PATH}/EXPECTED_BUILDS" && -d $SCRIPT_PATH/checkout/WebKitBuild/Release/Playwright.app ]]; then - DYLIB_PATH="$SCRIPT_PATH/checkout/WebKitBuild/Release" + elif [[ -f "${SCRIPT_PATH}/EXPECTED_BUILDS" && -d "$HOME/webkit/WebKitBuild/Release/Playwright.app" ]]; then + DYLIB_PATH="$HOME/webkit/WebKitBuild/Release" elif [[ -d $SCRIPT_PATH/Playwright.app ]]; then DYLIB_PATH="$SCRIPT_PATH" elif [[ -d $SCRIPT_PATH/WebKitBuild/Release/Playwright.app ]]; then @@ -36,11 +36,11 @@ function runLinux() { # the zip bundle wrapper already sets itself the needed env variables. if [[ -d $SCRIPT_PATH/$MINIBROWSER_FOLDER ]]; then MINIBROWSER="$SCRIPT_PATH/$MINIBROWSER_FOLDER/MiniBrowser" - elif [[ -d $SCRIPT_PATH/checkout/$BUILD_FOLDER ]]; then - LD_PATH="$SCRIPT_PATH/checkout/$BUILD_FOLDER/$DEPENDENCIES_FOLDER/Root/lib:$SCRIPT_PATH/checkout/$BUILD_FOLDER/Release/bin" - GIO_DIR="$SCRIPT_PATH/checkout/$BUILD_FOLDER/$DEPENDENCIES_FOLDER/Root/lib/gio/modules" - BUNDLE_DIR="$SCRIPT_PATH/checkout/$BUILD_FOLDER/Release/lib" - MINIBROWSER="$SCRIPT_PATH/checkout/$BUILD_FOLDER/Release/bin/MiniBrowser" + elif [[ -d $HOME/webkit/$BUILD_FOLDER ]]; then + LD_PATH="$HOME/webkit/$BUILD_FOLDER/$DEPENDENCIES_FOLDER/Root/lib:$SCRIPT_PATH/checkout/$BUILD_FOLDER/Release/bin" + GIO_DIR="$HOME/webkit/$BUILD_FOLDER/$DEPENDENCIES_FOLDER/Root/lib/gio/modules" + BUNDLE_DIR="$HOME/webkit/$BUILD_FOLDER/Release/lib" + MINIBROWSER="$HOME/webkit/$BUILD_FOLDER/Release/bin/MiniBrowser" elif [[ -f $SCRIPT_PATH/MiniBrowser ]]; then MINIBROWSER="$SCRIPT_PATH/MiniBrowser" elif [[ -d $SCRIPT_PATH/$BUILD_FOLDER ]]; then