devops: add script to generate shared object => package mapping (#3022)

We use this mapping to provide recommendations on which packages
to install on Linux distributions.

References #2745

utils/linux-browser-dependencies/.gitignore

@@ -0,0 +1,2 @@
+RUN_RESULT
+playwright.tar.gz

utils/linux-browser-dependencies/README.md

@@ -0,0 +1,28 @@
+# Mapping distribution libraries to package names
+
+Playwright requires a set of packages on Linux distribution for browsers to work.
+Before launching browser on Linux, Playwright uses `ldd` to make sure browsers have all
+dependencies met.
+
+If this is not the case, Playwright suggests users packages to install to
+meet the dependencies. This tool helps to maintain a map between package names
+and shared libraries it provides, per distribution.
+
+## Usage
+
+To generate a map of browser library to package name on Ubuntu:bionic:
+
+```sh
+$ ./run.sh ubuntu:bionic
+```
+
+Results will be saved to the `RUN_RESULT`.
+
+
+## How it works
+
+The script does the following:
+
+1. Launches docker with given linux distribution
+2. Installs playwright browsers inside the distribution
+3. For every dependency that Playwright browsers miss inside the distribution, uses `apt-file` to reverse-search package with the library.

utils/linux-browser-dependencies/inside_docker/list_dependencies.js

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const util = require('util');
+const path = require('path');
+const {spawn} = require('child_process');
+const browserPaths = require('playwright/lib/install/browserPaths.js');
+
+(async () => {
+  const allBrowsersPath = browserPaths.browsersPath();
+  const {stdout} = await runCommand('find', [allBrowsersPath, '-executable', '-type', 'f']);
+  // lddPaths - files we want to run LDD against.
+  const lddPaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh'));
+  // List of all shared libraries missing.
+  const missingDeps = new Set();
+  // Multimap: reverse-mapping from shared library to requiring file.
+  const depsToLddPaths = new Map();
+  await Promise.all(lddPaths.map(async lddPath => {
+    const deps = await missingFileDependencies(lddPath);
+    for (const dep of deps) {
+      missingDeps.add(dep);
+      let depsToLdd = depsToLddPaths.get(dep);
+      if (!depsToLdd) {
+        depsToLdd = new Set();
+        depsToLddPaths.set(dep, depsToLdd);
+      }
+      depsToLdd.add(lddPath);
+    }
+  }));
+  console.log(`==== MISSING DEPENDENCIES: ${missingDeps.size} ====`);
+  console.log([...missingDeps].sort().join('\n'));
+
+  console.log('{');
+  for (const dep of missingDeps) {
+    const packages = await findPackages(dep);
+    if (packages.length === 0) {
+      console.log(`  // UNRESOLVED: ${dep} `);
+      const depsToLdd = depsToLddPaths.get(dep);
+      for (const filePath of depsToLdd)
+        console.log(`  // - required by ${filePath}`);
+    } else if (packages.length === 1) {
+      console.log(`  "${dep}": "${packages[0]}",`);
+    } else {
+      console.log(`  "${dep}": ${JSON.stringify(packages)},`);
+    }
+  }
+  console.log('}');
+})();
+
+async function findPackages(libraryName) {
+  const {stdout} = await runCommand('apt-file', ['search', libraryName]);
+  if (!stdout.trim())
+    return [];
+  const libs = stdout.trim().split('\n').map(line => line.split(':')[0]);
+  return [...new Set(libs)];
+}
+
+async function fileDependencies(filePath) {
+  const {stdout} = await lddAsync(filePath);
+  const deps = stdout.split('\n').map(line => {
+    line = line.trim();
+    const missing = line.includes('not found');
+    const name = line.split('=>')[0].trim();
+    return {name, missing};
+  });
+  return deps;
+}
+
+async function missingFileDependencies(filePath) {
+  const deps = await fileDependencies(filePath);
+  return deps.filter(dep => dep.missing).map(dep => dep.name);
+}
+
+async function lddAsync(filePath) {
+  let LD_LIBRARY_PATH = [];
+  // Some shared objects inside browser sub-folders link against libraries that
+  // ship with the browser. We consider these to be included, so we want to account
+  // for them in the LD_LIBRARY_PATH.
+  for (let dirname = path.dirname(filePath); dirname !== '/'; dirname = path.dirname(dirname))
+    LD_LIBRARY_PATH.push(dirname);
+  return await runCommand('ldd', [filePath], {
+    cwd: path.dirname(filePath),
+    env: {
+      ...process.env,
+      LD_LIBRARY_PATH: LD_LIBRARY_PATH.join(':'),
+    },
+  });
+}
+
+function runCommand(command, args, options = {}) {
+  const childProcess = spawn(command, args, options);
+
+  return new Promise((resolve) => {
+    let stdout = '';
+    let stderr = '';
+    childProcess.stdout.on('data', data => stdout += data);
+    childProcess.stderr.on('data', data => stderr += data);
+    childProcess.on('close', (code) => {
+      resolve({stdout, stderr, code});
+    });
+  });
+}

utils/linux-browser-dependencies/inside_docker/process.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+set +x
+
+# Install Node.js
+
+apt-get update && apt-get install -y curl && \
+    curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
+    apt-get install -y nodejs
+
+# Install apt-file
+apt-get update && apt-get install -y apt-file && apt-file update
+
+# Install tip-of-tree playwright
+mkdir /root/tmp && cd /root/tmp && npm init -y && npm i /root/hostfolder/playwright.tar.gz
+
+cp /root/hostfolder/inside_docker/list_dependencies.js /root/tmp/list_dependencies.js
+
+node list_dependencies.js | tee /root/hostfolder/RUN_RESULT

utils/linux-browser-dependencies/run.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+set +x
+
+if [[ ($1 == '--help') || ($1 == '-h') ]]; then
+  echo "usage: $(basename $0) <image-name>"
+  echo
+  echo "List mapping between browser dependencies to package names and save results in RUN_RESULT file."
+  echo "Example:"
+  echo ""
+  echo "  $(basename $0) ubuntu:bionic"
+  echo ""
+  echo "NOTE: this requires Playwright dependencies to be installed with 'npm install'"
+  echo "      and Playwright itself being built with 'npm run build'"
+  echo ""
+  exit 0
+fi
+
+if [[ $# == 0 ]]; then
+  echo "ERROR: please provide base image name, e.g. 'ubuntu:bionic'"
+  exit 1
+fi
+
+function cleanup() {
+  rm -f "playwright.tar.gz"
+}
+
+trap "cleanup; cd $(pwd -P)" EXIT
+cd "$(dirname "$0")"
+
+# We rely on `./playwright.tar.gz` to download browsers into the docker image.
+node ../../packages/build_package.js playwright ./playwright.tar.gz
+
+docker run -v $PWD:/root/hostfolder --rm -it "$1" /root/hostfolder/inside_docker/process.sh
+