devops: support universal webkit builds (#15496)

browser_patches/checkout_build_archive_upload.sh

@@ -306,6 +306,13 @@ elif [[ "$BUILD_FLAVOR" == "firefox-beta-win64" ]]; then
 # ===========================
 #    WEBKIT COMPILATION
 # ===========================
+elif [[ "$BUILD_FLAVOR" == "webkit-universal" ]]; then
+  BROWSER_NAME="webkit"
+  EXTRA_BUILD_ARGS="--full --universal"
+  EXTRA_ARCHIVE_ARGS="--universal"
+  EXPECTED_HOST_OS="Ubuntu"
+  EXPECTED_HOST_OS_VERSION="20.04"
+  BUILD_BLOB_NAME="webkit-linux-universal.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-18.04" ]]; then
   BROWSER_NAME="webkit"
   EXTRA_BUILD_ARGS="--full"

browser_patches/webkit/EXPECTED_BUILDS

@@ -1,3 +1,4 @@
+webkit-linux-universal.zip
 webkit-ubuntu-18.04.zip
 webkit-ubuntu-20.04.zip
 webkit-ubuntu-20.04-arm64.zip

browser_patches/webkit/archive.sh

@@ -3,7 +3,7 @@ set -e
 set +x
 
 if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
-  echo "usage: $(basename "$0") [output-absolute-path]"
+  echo "usage: $(basename "$0") [output-absolute-path] [--universal]"
   echo
   echo "Generate distributable .zip archive from ./checkout folder that was previously built."
   echo
@@ -28,6 +28,11 @@ if ! [[ -d $(dirname "$ZIP_PATH") ]]; then
   exit 1
 fi
 
+IS_UNIVERSAL=""
+if [[ $2 == "--universal" ]]; then
+  IS_UNIVERSAL=1
+fi
+
 main() {
   if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then
     cd "${WK_CHECKOUT_PATH}"
@@ -52,7 +57,7 @@ main() {
 
 createZipForLinux() {
   # create a TMP directory to copy all necessary files
-  local tmpdir=$(mktemp -d -t webkit-deploy-XXXXXXXXXX)
+  local tmpdir=$(mktemp -d -p "$(pwd)/WebKitBuild" -t webkit-deploy-XXXXXXXXXX)
   mkdir -p "$tmpdir"
 
   # copy runner
@@ -62,15 +67,27 @@ createZipForLinux() {
 
   # Generate and unpack MiniBrowser bundles for each port
   for port in gtk wpe; do
-    WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/${port^^} Tools/Scripts/generate-bundle \
+    if [[ -n "${IS_UNIVERSAL}" ]]; then
-        --bundle=MiniBrowser --release \
+      Tools/Scripts/generate-bundle \
-        --platform=${port} --destination="${tmpdir}"
+          --syslibs=bundle-all \
-     unzip "${tmpdir}"/MiniBrowser_${port}_release.zip -d "${tmpdir}"/minibrowser-${port}
+          --bundle=MiniBrowser --release \
-     rm -f "${tmpdir}"/MiniBrowser_${port}_release.zip
+          --platform=${port} --destination="${tmpdir}"
+    else
+      WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/${port^^} \
+      Tools/Scripts/generate-bundle \
+          --bundle=MiniBrowser --release \
+          --platform=${port} --destination="${tmpdir}"
+    fi
+
+    unzip "${tmpdir}"/MiniBrowser_${port}_release.zip -d "${tmpdir}"/minibrowser-${port}
+    rm -f "${tmpdir}"/MiniBrowser_${port}_release.zip
   done
 
-  # tar resulting directory and cleanup TMP.
   cd "$tmpdir"
+  # de-duplicate common files: convert to relative symlinks identical files (same hash)
+  rdfind -deterministic true -makesymlinks true -makehardlinks false -makeresultsfile false .
+  symlinks -rc .
+  # zip resulting directory and cleanup TMP.
   zip --symlinks -r "$ZIP_PATH" ./
   cd -
   rm -rf "$tmpdir"

browser_patches/webkit/build.sh

@@ -7,9 +7,18 @@ cd "$(dirname "$0")"
 SCRIPT_FOLDER="$(pwd -P)"
 source "${SCRIPT_FOLDER}/../utils.sh"
 
+# On Linux, Universal build uses Flatpak rather then JHBuild
+# and packs into a universal binary that can run on any linux
+# distribution.
+IS_UNIVERSAL_BUILD=""
+
 build_gtk() {
-  if ! [[ -d ./WebKitBuild/GTK/DependenciesGTK ]]; then
+  if [[ -z "${IS_UNIVERSAL_BUILD}" && ! -d "./WebKitBuild/GTK/DependenciesGTK" ]]; then
-    yes | WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK DEBIAN_FRONTEND=noninteractive ./Tools/Scripts/update-webkitgtk-libs
+    yes | WEBKIT_JHBUILD=1 \
+          WEBKIT_JHBUILD_MODULESET=minimal \
+          WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK \
+          DEBIAN_FRONTEND=noninteractive \
+          ./Tools/Scripts/update-webkitgtk-libs
   fi
   local CMAKE_ARGS=(
     --cmakeargs=-DENABLE_INTROSPECTION=OFF
@@ -18,12 +27,23 @@ build_gtk() {
   if [[ -n "${EXPORT_COMPILE_COMMANDS}" ]]; then
     CMAKE_ARGS+=("--cmakeargs=-DCMAKE_EXPORT_COMPILE_COMMANDS=1")
   fi
-  WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  else
+    WEBKIT_JHBUILD=1 \
+    WEBKIT_JHBUILD_MODULESET=minimal \
+    WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK \
+    ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  fi
 }
 
 build_wpe() {
-  if ! [[ -d ./WebKitBuild/WPE/DependenciesWPE ]]; then
+  if [[ -z "${IS_UNIVERSAL_BUILD}" && ! -d "./WebKitBuild/WPE/DependenciesWPE" ]]; then
-    yes | WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE DEBIAN_FRONTEND=noninteractive ./Tools/Scripts/update-webkitwpe-libs
+    yes | WEBKIT_JHBUILD=1 \
+          WEBKIT_JHBUILD_MODULESET=minimal \
+          WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE \
+          DEBIAN_FRONTEND=noninteractive \
+          ./Tools/Scripts/update-webkitwpe-libs
   fi
   local CMAKE_ARGS=(
     --cmakeargs=-DENABLE_COG=OFF
@@ -34,14 +54,37 @@ build_wpe() {
   if [[ -n "${EXPORT_COMPILE_COMMANDS}" ]]; then
     CMAKE_ARGS+=("--cmakeargs=-DCMAKE_EXPORT_COMPILE_COMMANDS=1")
   fi
-  WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  else
+    WEBKIT_JHBUILD=1 \
+    WEBKIT_JHBUILD_MODULESET=minimal \
+    WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE \
+    ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  fi
 }
 
 ensure_linux_deps() {
+  SUDO="" ; [ $UID -ne 0 ] && SUDO="sudo"
+
+  # These two packages are needed to de-duplicate files on the GTK+WPE bundle and reduce its size.
+  DEBIAN_FRONTEND=noninteractive ${SUDO} apt-get install -y symlinks rdfind
+
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    DEBIAN_FRONTEND=noninteractive ${SUDO} apt-get install -y flatpak
+  fi
+
   yes | DEBIAN_FRONTEND=noninteractive ./Tools/gtk/install-dependencies
   yes | DEBIAN_FRONTEND=noninteractive ./Tools/wpe/install-dependencies
-  yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/update-webkitwpe-libs
+  if [[ -z "${IS_UNIVERSAL_BUILD}" ]]; then
-  yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/update-webkitgtk-libs
+    # In non-universal build install JHBuild deps.
+    yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/update-webkitwpe-libs
+    yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/update-webkitgtk-libs
+  else
+    yes | ./Tools/Scripts/update-webkitwpe-libs
+    yes | ./Tools/Scripts/update-webkitgtk-libs
+  fi
 }
 
 if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then
@@ -55,29 +98,57 @@ if is_mac; then
   selectXcodeVersionOrDie $(node "$SCRIPT_FOLDER/../get_xcode_version.js" webkit)
   ./Tools/Scripts/build-webkit --release --touch-events --orientation-events
 elif is_linux; then
-  if [[ $# == 0 || (-z "$1") ]]; then
+  args=("$@")
-    echo
+  IS_FULL=""
-    echo BUILDING: GTK and WPE
+  BUILD_GTK=""
-    echo
+  BUILD_WPE=""
-    build_wpe
+  for ((i="${#args[@]}"-1; i >= 0; --i)); do
-    build_gtk
+      case ${args[i]} in
-  elif [[ "$1" == "--full" ]]; then
+          --full) IS_FULL="1"; unset args[i]; ;;
-    echo
+          --gtk) BUILD_GTK="1"; unset args[i]; ;;
-    echo BUILDING: GTK and WPE
+          --wpe) BUILD_WPE="1"; unset args[i]; ;;
-    echo
+          --universal) IS_UNIVERSAL_BUILD="1"; unset args[i]; ;;
+      esac
+  done
+
+  # if neither gtk nor wpe is requested then build both.
+  if [[ -z "${BUILD_GTK}" && -z "${BUILD_WPE}" ]]; then
+    BUILD_GTK="1"
+    BUILD_WPE="1"
+  fi
+
+  echo "== BUILD CONFIGURATION =="
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    echo "- universal build: YES"
+  else
+    echo "- universal build: NO"
+  fi
+  if [[ -n "${IS_FULL}" ]]; then
+    echo "- install dependencies: YES"
+  else
+    echo "- install dependencies: NO"
+  fi
+  if [[ -n "${BUILD_GTK}" ]]; then
+    echo "- build GTK: YES"
+  else
+    echo "- build GTK: NO"
+  fi
+  if [[ -n "${BUILD_WPE}" ]]; then
+    echo "- build WPE: YES"
+  else
+    echo "- build WPE: NO"
+  fi
+
+  if [[ -n "${IS_FULL}" ]]; then
     ensure_linux_deps
+  fi
+
+  if [[ -n "${BUILD_WPE}" ]]; then
     build_wpe
+  fi
+
+  if [[ -n "${BUILD_GTK}" ]]; then
     build_gtk
-  elif [[ "$1" == "--gtk" ]]; then
-    echo
-    echo BUILDING: GTK
-    echo
-    build_gtk
-  elif [[ "$1" == "--wpe" ]]; then
-    echo
-    echo BUILDING: WPE
-    echo
-    build_wpe
   fi
 elif is_win; then
   /c/Windows/System32/cmd.exe "/c $(cygpath -w "${SCRIPT_FOLDER}"/buildwin.bat)"