devops: improve chromium toolchain (#12897)

browser_patches/checkout_build_archive_upload.sh

@@ -91,35 +91,30 @@ elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win64" ]]; then
 # ===========================
 elif [[ "$BUILD_FLAVOR" == "chromium-win64" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--compile-win64 --full"
-  EXTRA_ARCHIVE_ARGS="--compile-win64"
+  EXTRA_BUILD_ARGS="--full"
   EXPECTED_HOST_OS="MINGW"
   BUILD_BLOB_NAME="chromium-win64.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-mac" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--compile-mac --full"
-  EXTRA_ARCHIVE_ARGS="--compile-mac"
+  EXTRA_BUILD_ARGS="--full"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="12.2"
   BUILD_BLOB_NAME="chromium-mac.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-mac-arm64" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--compile-mac-arm64 --full"
-  EXTRA_ARCHIVE_ARGS="--compile-mac-arm64"
+  EXTRA_BUILD_ARGS="--arm64 --full"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="12.2"
   BUILD_BLOB_NAME="chromium-mac-arm64.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-linux" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--compile-linux --full"
-  EXTRA_ARCHIVE_ARGS="--compile-linux"
+  EXTRA_BUILD_ARGS="--full"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-linux.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-linux-arm64" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--compile-linux-arm64 --full"
-  EXTRA_ARCHIVE_ARGS="--compile-linux-arm64"
+  EXTRA_BUILD_ARGS="--arm64 --full"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="20.04"
   BUILD_BLOB_NAME="chromium-linux-arm64.zip"
@@ -130,16 +125,14 @@ elif [[ "$BUILD_FLAVOR" == "chromium-linux-arm64" ]]; then
 elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-win64" ]]; then
   BROWSER_NAME="chromium"
   BROWSER_DISPLAY_NAME="chromium-with-symbols"
-  EXTRA_BUILD_ARGS="--compile-win64 --symbols --full"
-  EXTRA_ARCHIVE_ARGS="--compile-win64"
+  EXTRA_BUILD_ARGS="--symbols --full"
   EXPECTED_HOST_OS="MINGW"
   BUILD_BLOB_NAME="chromium-with-symbols-win64.zip"
   BUILDS_LIST="EXPECTED_BUILDS_WITH_SYMBOLS"
 elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-mac" ]]; then
   BROWSER_NAME="chromium"
   BROWSER_DISPLAY_NAME="chromium-with-symbols"
-  EXTRA_BUILD_ARGS="--compile-mac --symbols --full"
-  EXTRA_ARCHIVE_ARGS="--compile-mac"
+  EXTRA_BUILD_ARGS="--symbols --full"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="12.2"
   BUILD_BLOB_NAME="chromium-with-symbols-mac.zip"
@@ -147,8 +140,7 @@ elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-mac" ]]; then
 elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-mac-arm64" ]]; then
   BROWSER_NAME="chromium"
   BROWSER_DISPLAY_NAME="chromium-with-symbols"
-  EXTRA_BUILD_ARGS="--compile-mac-arm64 --symbols --full"
-  EXTRA_ARCHIVE_ARGS="--compile-mac-arm64"
+  EXTRA_BUILD_ARGS="--arm64 --symbols --full"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="12.2"
   BUILD_BLOB_NAME="chromium-with-symbols-mac-arm64.zip"
@@ -156,8 +148,7 @@ elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-mac-arm64" ]]; then
 elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-linux" ]]; then
   BROWSER_NAME="chromium"
   BROWSER_DISPLAY_NAME="chromium-with-symbols"
-  EXTRA_BUILD_ARGS="--compile-linux --symbols --full"
-  EXTRA_ARCHIVE_ARGS="--compile-linux"
+  EXTRA_BUILD_ARGS="--symbols --full"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-with-symbols-linux.zip"
@@ -165,8 +156,7 @@ elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-linux" ]]; then
 elif [[ "$BUILD_FLAVOR" == "chromium-with-symbols-linux-arm64" ]]; then
   BROWSER_NAME="chromium"
   BROWSER_DISPLAY_NAME="chromium-with-symbols-arm64"
-  EXTRA_BUILD_ARGS="--compile-linux-arm64 --symbols --full"
-  EXTRA_ARCHIVE_ARGS="--compile-linux-arm64"
+  EXTRA_BUILD_ARGS="--arm64 --symbols --full"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="20.04"
   BUILD_BLOB_NAME="chromium-with-symbols-linux-arm64.zip"

browser_patches/chromium/archive.sh

@@ -6,7 +6,6 @@ trap "cd $(pwd -P)" EXIT
 cd "$(dirname "$0")"
 SCRIPT_PATH=$(pwd -P)
 
-main() {
 if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
   echo "usage: $(basename "$0") [output-absolute-path]"
   echo
@@ -34,21 +33,6 @@ main() {
   exit 1
 fi
 
-  BUILD_TYPE=$2
-  if [[ "${BUILD_TYPE}" == "--compile"* ]]; then
-    archive_compiled_chromium "${BUILD_TYPE}"
-  elif [[ "${BUILD_TYPE}" == "--mirror"* ]]; then
-    archive_mirrored_chromium "${BUILD_TYPE}"
-  else
-    echo "ERROR: unknown build type - ${BUILD_TYPE}"
-    exit 1
-  fi
-
-  cd "${SCRIPT_PATH}"
-  cp output/build.zip "$ZIP_PATH"
-}
-
-function archive_compiled_chromium() {
 if [[ -z "${CR_CHECKOUT_PATH}" ]]; then
   CR_CHECKOUT_PATH="$HOME/chromium"
 fi
@@ -60,24 +44,20 @@ function archive_compiled_chromium() {
 CHROMIUM_FOLDER_NAME=""
 CHROMIUM_FILES_TO_ARCHIVE=()
 
-  if [[ $1 == "--compile-mac"* ]]; then
+if [[ $(uname) == "Darwin" ]]; then
   CHROMIUM_FOLDER_NAME="chrome-mac"
   IFS=$'\n' CHROMIUM_FILES_TO_ARCHIVE=($(node "${SCRIPT_PATH}/compute_files_to_archive.js" "${CR_CHECKOUT_PATH}/src/infra/archive_config/mac-archive-rel.json"))
   unset IFS
-  elif [[ $1 == "--compile-linux" ]]; then
+elif [[ $(uname) == "Linux" ]]; then
   CHROMIUM_FOLDER_NAME="chrome-linux"
   IFS=$'\n' CHROMIUM_FILES_TO_ARCHIVE=($(node "${SCRIPT_PATH}/compute_files_to_archive.js" "${CR_CHECKOUT_PATH}/src/infra/archive_config/linux-archive-rel.json"))
   unset IFS
-  elif [[ $1 == "--compile-linux-arm64" ]]; then
-    CHROMIUM_FOLDER_NAME="chrome-linux"
-    IFS=$'\n' CHROMIUM_FILES_TO_ARCHIVE=($(node "${SCRIPT_PATH}/compute_files_to_archive.js" "${CR_CHECKOUT_PATH}/src/infra/archive_config/linux-archive-rel.json"))
-    unset IFS
-  elif [[ $1 == "--compile-win64" ]]; then
+elif [[ $(uname) == "MINGW" ]]; then
   CHROMIUM_FOLDER_NAME="chrome-win"
   IFS=$'\n\r' CHROMIUM_FILES_TO_ARCHIVE=($(node "${SCRIPT_PATH}/compute_files_to_archive.js" "${CR_CHECKOUT_PATH}/src/infra/archive_config/win-archive-rel.json"))
   unset IFS
 else
-    echo "ERROR: unknown command, use --help for details"
+  echo "ERROR: unsupported platform - $(uname)"
   exit 1
 fi
 
@@ -98,7 +78,7 @@ function archive_compiled_chromium() {
   $COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/${file}" "output/${CHROMIUM_FOLDER_NAME}/${file}"
 done
 
-  if [[ $1 == "--compile-win"* ]]; then
+if [[ $(uname) == "MINGW" ]]; then
   $COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/"*.manifest "output/${CHROMIUM_FOLDER_NAME}/"
   mkdir -p "output/${CHROMIUM_FOLDER_NAME}/locales"
   $COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/locales/"*.pak "output/${CHROMIUM_FOLDER_NAME}/locales/"
@@ -106,35 +86,6 @@ function archive_compiled_chromium() {
 
 cd output
 zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
-}
 
-archive_mirrored_chromium() {
-  cd "${SCRIPT_PATH}/output"
-
-  CHROMIUM_FOLDER_NAME=""
-  CHROMIUM_FILES_TO_REMOVE=()
-
-  PLATFORM="$1"
-  if [[ "${PLATFORM}" == "--mirror-win32" ]]; then
-    CHROMIUM_FOLDER_NAME="chrome-win"
-    CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
-  elif [[ "${PLATFORM}" == "--mirror-win64" ]]; then
-    CHROMIUM_FOLDER_NAME="chrome-win"
-    CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
-  elif [[ "${PLATFORM}" == "--mirror-mac" ]]; then
-    CHROMIUM_FOLDER_NAME="chrome-mac"
-  elif [[ "${PLATFORM}" == "--mirror-linux" ]]; then
-    CHROMIUM_FOLDER_NAME="chrome-linux"
-  else
-    echo "ERROR: unknown platform to build: $PLATFORM"
-    exit 1
-  fi
-
-  for file in ${CHROMIUM_FILES_TO_REMOVE[@]}; do
-    rm -f "${file}"
-  done
-
-  zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
-}
-
-main "$@"
+cd "${SCRIPT_PATH}"
+cp output/build.zip "$ZIP_PATH"

browser_patches/chromium/build.sh

@@ -4,29 +4,43 @@ set +x
 
 trap "cd $(pwd -P)" EXIT
 cd "$(dirname "$0")"
+SCRIPT_FOLDER=$(pwd -P)
 
 USAGE=$(cat<<EOF
-  usage: $(basename "$0") [--compile-mac-arm64|--compile-linux|--compile-linux-arm64|--compile-win64|--compile-mac] [--symbols] [--full]
+  usage: $(basename "$0") [--arm64] [--symbols] [--full] [--goma] <custom targets to compile>
+
+  --arm64     cross-compile for arm64
+  --symbols   compile with symbols
+  --full      install build dependencies
+  --goma      use goma when compiling. Make sure to pre-start goma client beforehand with './goma.sh start'.
+
+  On Linux & MacOS, it is possible to specify custom compilation targets:
+
+    ./build.sh --goma blink_tests
 
-  Compiles chromium.
 EOF
 )
 
-SCRIPT_FOLDER=$(pwd -P)
 source "${SCRIPT_FOLDER}/../utils.sh"
 
-main() {
 if [[ $1 == "--help" || $1 == "-h" ]]; then
   echo "$USAGE"
   exit 0
-  elif [[ $1 == "--compile"* ]]; then
-    compile_chromium "$1" "$2" "$3"
-  else
-    echo "ERROR: unknown first argument. Use --help for details."
-    exit 1
 fi
-}
 
+args=("$@")
+IS_ARM64=""
+IS_SYMBOLS_BUILD=""
+IS_FULL=""
+USE_GOMA=""
+for ((i=0; i<="${#args[@]}"; ++i)); do
+    case ${args[i]} in
+        --arm64) IS_ARM64="1"; unset args[i]; ;;
+        --symbols) IS_SYMBOLS_BUILD="1"; unset args[i]; ;;
+        --full) IS_FULL="1"; unset args[i]; ;;
+        --goma) USE_GOMA="1"; unset args[i]; ;;
+    esac
+done
 
 compile_chromium() {
   if [[ -z "${CR_CHECKOUT_PATH}" ]]; then
@@ -40,15 +54,9 @@ compile_chromium() {
 
   source "${SCRIPT_FOLDER}/ensure_depot_tools.sh"
 
-  if [[ $1 == "--compile-mac"* ]]; then
+  if [[ $(uname) == "Darwin" ]]; then
     # As of Feb, 2022 Chromium mac compilation requires Xcode13.2
     selectXcodeVersionOrDie "13.2"
-    # As of Jan, 2021 Chromium mac compilation is only possible on Intel macbooks.
-    # See https://chromium.googlesource.com/chromium/src.git/+/main/docs/mac_arm64.md
-    if [[ $1 == "--compile-mac-arm64" && $(uname -m) != "x86_64" ]]; then
-      echo "ERROR: chromium mac arm64 compilation is (ironically) only supported on Intel Macbooks"
-      exit 1
-    fi
   fi
 
   cd "${CR_CHECKOUT_PATH}/src"
@@ -57,25 +65,18 @@ compile_chromium() {
   mkdir -p "./out/Default"
   echo "is_debug = false" > ./out/Default/args.gn
   echo "dcheck_always_on = false" >> ./out/Default/args.gn
-  if [[ $2 == "--symbols" || $3 == "--symbols" ]]; then
+  if [[ -n "${IS_SYMBOLS_BUILD}" ]]; then
     echo "symbol_level = 1" >> ./out/Default/args.gn
   else
     echo "symbol_level = 0" >> ./out/Default/args.gn
   fi
 
-  if [[ $1 == "--compile-mac-arm64" ]]; then
-    echo 'target_cpu = "arm64"' >> ./out/Default/args.gn
-  elif [[ $1 == "--compile-linux-arm64" ]]; then
+  if [[ -n "${IS_ARM64}" ]]; then
     echo 'target_cpu = "arm64"' >> ./out/Default/args.gn
   fi
 
   if [[ ! -z "$USE_GOMA" ]]; then
-    PLAYWRIGHT_GOMA_PATH="${SCRIPT_FOLDER}/electron-build-tools/third_party/goma"
-    if [[ $1 == "--compile-win"* ]]; then
-      PLAYWRIGHT_GOMA_PATH=$(cygpath -w "${PLAYWRIGHT_GOMA_PATH}")
-    fi
-    echo 'use_goma = true' >> ./out/Default/args.gn
-    echo "goma_dir = \"${PLAYWRIGHT_GOMA_PATH}\"" >> ./out/Default/args.gn
+    "${SCRIPT_FOLDER}/goma.sh" args >> ./out/Default/args.gn
   fi
   echo 'enable_nacl = false' >> ./out/Default/args.gn
 
@@ -83,17 +84,23 @@ compile_chromium() {
   cat ./out/Default/args.gn
   echo "===== ======= ====="
 
-  if [[ $2 == "--full" || $3 == "--full" ]]; then
-    if [[ $(uname) == "--compile-linux" ]]; then
-      ./build/install-build-deps.sh
-    elif [[ $1 == "--compile-linux-arm64" ]]; then
+  if [[ -n "$IS_FULL" ]]; then
+    if [[ $(uname) == "Linux" ]]; then
       ./build/install-build-deps.sh
+      if [[ -n "$IS_ARM64" ]]; then
         # Install sysroot image, see https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/linux/chromium_arm.md
         ./build/linux/sysroot_scripts/install-sysroot.py --arch=arm64
       fi
     fi
+  fi
 
-  if [[ $1 == "--compile-win"* ]]; then
+  TARGETS="$@"
+  if [[ $(uname) == "MINGW" ]]; then
+    if [[ -n "$TARGETS" ]]; then
+      echo "ERROR: cannot compile custom targets on windows yet."
+      echo "Requested to compile chromium targets - ${TARGETS}"
+      exit 1
+    fi
     if [[ -z "$USE_GOMA" ]]; then
       /c/Windows/System32/cmd.exe "/c $(cygpath -w "${SCRIPT_FOLDER}"/buildwin.bat)"
     else
@@ -101,11 +108,13 @@ compile_chromium() {
     fi
   else
     gn gen out/Default
-    if [[ $1 == "--compile-linux"* ]]; then
+    if [[ -z "$TARGETS" ]]; then
+      if [[ $(uname) == "Linux" ]]; then
         TARGETS="chrome chrome_sandbox clear_key_cdm"
       else
         TARGETS="chrome"
       fi
+    fi
     if [[ -z "$USE_GOMA" ]]; then
       autoninja -C out/Default $TARGETS
     else
@@ -114,4 +123,4 @@ compile_chromium() {
   fi
 }
 
-main "$1" "$2" "$3"
+compile_chromium "${args[@]}"

browser_patches/chromium/clean.sh

@@ -13,5 +13,3 @@ fi
 if [[ ! -d "${CR_CHECKOUT_PATH}/src" ]]; then
   rm -rf "${CR_CHECKOUT_PATH}/src/out"
 fi
-
-

browser_patches/chromium/goma.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 set -e
-set -x
+set +x
 
 trap "cd $(pwd -P)" EXIT
 cd "$(dirname "$0")"
+SCRIPT_FOLDER=$(pwd -P)
 
 ELECTRON_BUILD_TOOLS_REQUIRED_VERSION=6ba8962529c37727a778691b89c92ab0eb1d9d87
 if [[ -d ./electron-build-tools ]]; then
@@ -31,15 +32,29 @@ cd electron-build-tools/third_party/goma
 
 export GOMA_START_COMPILER_PROXY=true
 
+function print_gn_args() {
+  PLAYWRIGHT_GOMA_PATH="${SCRIPT_FOLDER}/electron-build-tools/third_party/goma"
+  if [[ $(uname) == MINGW* ]]; then
+    PLAYWRIGHT_GOMA_PATH=$(cygpath -w "${PLAYWRIGHT_GOMA_PATH}")
+  fi
+  echo 'use_goma = true'
+  echo "goma_dir = \"${PLAYWRIGHT_GOMA_PATH}\""
+}
+
 if [[ $1 == "--help" ]]; then
   echo "$(basename "$0") [login|start|stop|--help]"
   exit 0
+elif [[ $1 == "args" ]]; then
+  print_gn_args
 elif [[ $1 == "login" ]]; then
   if [[ $(uname) == "MINGW"* ]]; then
     /c/Windows/System32/cmd.exe "/c $(cygpath -w $(pwd)/goma_auth.bat) login"
   else
     python ./goma_auth.py login
   fi
+  echo
+  echo "Congratulation! Goma is logged in!"
+  echo "run '$(basename "$0") start' to launch goma client"
 elif [[ $1 == "start" ]]; then
   # We have to prefix ENV with `PLAYWRIGHT` since `GOMA_` env variables
   # have special treatment by goma.
@@ -56,6 +71,15 @@ elif [[ $1 == "start" ]]; then
   else
     python ./goma_ctl.py ensure_start
   fi
+  set +x
+  echo
+  echo "Congratulatons! Goma is running!"
+  echo
+  echo "Add the following gn args to use goma:"
+  echo
+  echo "===== args.gn ====="
+  print_gn_args
+  echo "===== ======= ====="
 elif [[ $1 == "stop" ]]; then
   if [[ $(uname) == "MINGW"* ]]; then
     /c/Windows/System32/cmd.exe "/c $(cygpath -w $(pwd)/goma_ctl.bat) stop"