[Reland] Roll sysroots

> Also needed is [1] in webrtc, which fixes some build warnings. Could
> land in parallel.
>
> [1] https://webrtc-review.googlesource.com/c/src/+/156980
>
> BUG=1012850
>
> Change-Id: Ia24e490db5e71cf1add0e10f4c1264aefb69143c
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1859935
> Auto-Submit: Thomas Anderson <thomasanderson@chromium.org>
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> Commit-Queue: Thomas Anderson <thomasanderson@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#705589}

BUG=1012850
R=dpranke

Change-Id: I1a3113201042d09931c0fe1ec68fe494b1b70df2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1862623
Auto-Submit: Thomas Anderson <thomasanderson@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Thomas Anderson <thomasanderson@chromium.org>
Cr-Commit-Position: refs/heads/master@{#706370}

build/linux/sysroot_scripts/generate_debian_archive_unstable_gpg.sh

+#!/bin/bash
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -o nounset
+set -o errexit
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+KEYS=(
+    # Debian Archive Automatic Signing Key (10/buster)
+    "DC30D7C23CBBABEE"
+    # Debian Security Archive Automatic Signing Key (10/buster)
+    "4DFAB270CAA96DFA"
+    # Debian Archive Automatic Signing Key (10/buster)
+    "DC30D7C23CBBABEE"
+    # Jessie Stable Release Key
+    "CBF8D6FD518E17E1"
+    # Debian Archive Automatic Signing Key (7.0/wheezy)
+    "8B48AD6246925553"
+    # Debian Archive Automatic Signing Key (8/jessie)
+    "7638D0442B90D010"
+    # Debian Security Archive Automatic Signing Key (8/jessie)
+    "9D6D8F6BC857C906"
+    # Debian Archive Automatic Signing Key (9/stretch)
+    "E0B11894F66AEC98"
+    # Debian Security Archive Automatic Signing Key (9/stretch)
+    "EDA0D2388AE22BA9"
+    # Debian Stable Release Key (9/stretch)
+    "EF0F382A1A7B6500"
+)
+
+gpg --recv-keys ${KEYS[@]}
+gpg --output "${SCRIPT_DIR}/debian_archive_unstable.gpg" --export ${KEYS[@]}

build/linux/sysroot_scripts/sysroot-creator-sid.sh

@@ -14,16 +14,18 @@ DIST=sid
 # like libc6 would take precedence over the sid (released) versions.  While this
 # may be useful for certain kinds of development, the standard sysroots should
 # continue to be shipped only with released packages.
+# Also keep "stretch" before "sid" and "experimental".  For now, it's needed to
+# bring back libgnome-keyring-dev which has since been deprecated and removed
+# from sid.  It will be needed until gnome keyring is removed
+# (http://crbug.com/466975 and http://crbug.com/355223).
 ARCHIVE_URL="https://snapshot.debian.org/archive/debian"
-ARCHIVE_TIMESTAMP=20181214T150526Z
-APT_SOURCES_LIST="${ARCHIVE_URL}/${ARCHIVE_TIMESTAMP}/ experimental main
+ARCHIVE_TIMESTAMP=20191015T084824Z
+APT_SOURCES_LIST="${ARCHIVE_URL}/${ARCHIVE_TIMESTAMP}/ stretch main
+${ARCHIVE_URL}/${ARCHIVE_TIMESTAMP}/ experimental main
 ${ARCHIVE_URL}/${ARCHIVE_TIMESTAMP}/ sid main"
 
-# gpg keyring file generated using:
-#   export KEYS="518E17E1 46925553 2B90D010 C857C906 F66AEC98 8AE22BA9 1A7B6500"
-#   gpg --recv-keys $KEYS
-#   gpg --output ./debian-archive-sid-stable.gpg --export $KEYS
-KEYRING_FILE="${SCRIPT_DIR}/debian-archive-sid-stable.gpg"
+# gpg keyring file generated using generate_debian_archive_unstable.sh
+KEYRING_FILE="${SCRIPT_DIR}/debian_archive_unstable.gpg"
 
 HAS_ARCH_AMD64=1
 HAS_ARCH_I386=1
@@ -57,11 +59,13 @@ DEBIAN_PACKAGES="\
   libaudit1
   libavahi-client3
   libavahi-common3
+  libblkid-dev
   libblkid1
   libbluetooth-dev
   libbluetooth3
   libbrlapi-dev
-  libbrlapi0.6
+  libbrlapi0.7
+  libbrotli1
   libbsd0
   libc6
   libc6-dev
@@ -115,7 +119,7 @@ DEBIAN_PACKAGES="\
   libfribidi0
   libgbm-dev
   libgbm1
-  libgcc-6-dev
+  libgcc-7-dev
   libgcc1
   libgcrypt20
   libgcrypt20-dev
@@ -162,7 +166,6 @@ DEBIAN_PACKAGES="\
   libice6
   libicu-le-hb0
   libicu63
-  libicu60
   libidl-2-0
   libidn11
   libidn2-0
@@ -189,6 +192,7 @@ DEBIAN_PACKAGES="\
   liblzo2-2
   libminizip-dev
   libminizip1
+  libmount-dev
   libmount1
   libnettle6
   libnspr4
@@ -201,8 +205,6 @@ DEBIAN_PACKAGES="\
   libopengl0
   libopus-dev
   libopus0
-  liborbit-2-0
-  liborbit2
   libp11-kit0
   libpam0g
   libpam0g-dev
@@ -216,6 +218,11 @@ DEBIAN_PACKAGES="\
   libpci3
   libpciaccess0
   libpcre16-3
+  libpcre2-16-0
+  libpcre2-32-0
+  libpcre2-8-0
+  libpcre2-dev
+  libpcre2-posix0
   libpcre3
   libpcre3-dev
   libpcre32-3
@@ -231,10 +238,13 @@ DEBIAN_PACKAGES="\
   libpulse-dev
   libpulse-mainloop-glib0
   libpulse0
-  libre2-4
+  libre2-5
   libre2-dev
   librest-0.7-0
+  libselinux1-dev
   libselinux1
+  libsepol1-dev
+  libsepol1
   libsm6
   libsnappy-dev
   libsnappy1v5
@@ -246,9 +256,8 @@ DEBIAN_PACKAGES="\
   libspeechd2
   libsqlite3-0
   libssl-dev
-  libssl1.0.2
   libssl1.1
-  libstdc++-6-dev
+  libstdc++-7-dev
   libstdc++6
   libsystemd0
   libtasn1-6
@@ -268,7 +277,7 @@ DEBIAN_PACKAGES="\
   libvorbis0a
   libvorbisenc2
   libvpx-dev
-  libvpx5
+  libvpx6
   libvulkan1
   libwayland-client0
   libwayland-cursor0
@@ -366,7 +375,7 @@ DEBIAN_PACKAGES_AMD64="
 "
 
 DEBIAN_PACKAGES_X86="
-  libasan3
+  libasan4
   libcilkrts5
   libdrm-intel1
   libitm1
@@ -376,7 +385,8 @@ DEBIAN_PACKAGES_X86="
 "
 
 DEBIAN_PACKAGES_ARM="
-  libasan3
+  libasan4
+  libcilkrts5
   libdrm-etnaviv1
   libdrm-exynos1
   libdrm-freedreno1
@@ -386,18 +396,21 @@ DEBIAN_PACKAGES_ARM="
 "
 
 DEBIAN_PACKAGES_ARM64="
-  libasan3
+  libasan4
   libdrm-etnaviv1
   libdrm-freedreno1
   libdrm-tegra0
   libgmp10
   libitm1
+  liblsan0
   libthai0
+  libtsan0
   libubsan0
 "
 
 DEBIAN_PACKAGES_ARMEL="
-  libasan3
+  libasan4
+  libcilkrts5
   libdrm-exynos1
   libdrm-freedreno1
   libdrm-omap1
@@ -405,4 +418,7 @@ DEBIAN_PACKAGES_ARMEL="
   libubsan0
 "
 
+DEBIAN_PACKAGES_MIPS64EL="
+"
+
 . "${SCRIPT_DIR}/sysroot-creator.sh"

build/linux/sysroot_scripts/sysroot-creator.sh

@@ -104,7 +104,10 @@ DownloadOrCopyNonUniqueFilename() {
   local hash="$(echo "$url" | sha256sum | cut -d' ' -f1)"
 
   DownloadOrCopy "${url}" "${dest}.${hash}"
-  cp "${dest}.${hash}" "$dest"
+  # cp the file to prevent having to redownload it, but mv it to the
+  # final location so that it's atomic.
+  cp "${dest}.${hash}" "${dest}.$$"
+  mv "${dest}.$$" "${dest}"
 }
 
 DownloadOrCopy() {
@@ -119,10 +122,25 @@ DownloadOrCopy() {
     SubBanner "downloading from $1 -> $2"
     # Appending the "$$" shell pid is necessary here to prevent concurrent
     # instances of sysroot-creator.sh from trying to write to the same file.
-    # --create-dirs is added in case there are slashes in the filename, as can
-    # happen with the "debian/security" release class.
-    curl -L "$1" --create-dirs -o "${2}.partial.$$"
-    mv "${2}.partial.$$" $2
+    local temp_file="${2}.partial.$$"
+    # curl --retry doesn't retry when the page gives a 4XX error, so we need to
+    # manually rerun.
+    for i in {1..10}; do
+      # --create-dirs is added in case there are slashes in the filename, as can
+      # happen with the "debian/security" release class.
+      local http_code=$(curl -L "$1" --create-dirs -o "${temp_file}" \
+                        -w "%{http_code}")
+      if [ ${http_code} -eq 200 ]; then
+        break
+      fi
+      echo "Bad HTTP code ${http_code} when downloading $1"
+      rm -f "${temp_file}"
+      sleep $i
+    done
+    if [ ! -f "${temp_file}" ]; then
+      exit 1
+    fi
+    mv "${temp_file}" $2
   else
     SubBanner "copying from $1"
     cp "$1" "$2"
@@ -240,7 +258,6 @@ GeneratePackageListCommon() {
   local packages="$3"
 
   local dists="${DIST} ${DIST_UPDATES:-}"
-  local repos="main ${REPO_EXTRA:-}"
 
   local list_base="${BUILD_DIR}/Packages.${DIST}_${arch}"
   > "${list_base}"  # Create (or truncate) a zero-length file.
@@ -337,12 +354,27 @@ HacksAndPatchesCommon() {
   nm -D --defined-only --with-symbol-versions "${libc_so}" | \
     "${SCRIPT_DIR}/find_incompatible_glibc_symbols.py" >> "${glob_h}"
 
+  # fcntl64() was introduced in glibc 2.28.  Make sure to use fcntl() instead.
+  local fcntl_h="${INSTALL_ROOT}/usr/include/fcntl.h"
+  sed -i '{N; s/#ifndef \(__USE_FILE_OFFSET64\nextern int fcntl\)/#ifdef \1/}' \
+      "${fcntl_h}"
+  # On i386, fcntl() was updated in glibc 2.28.
+  nm -D --defined-only --with-symbol-versions "${libc_so}" | \
+    "${SCRIPT_DIR}/find_incompatible_glibc_symbols.py" >> "${fcntl_h}"
+
   # This is for chrome's ./build/linux/pkg-config-wrapper
   # which overwrites PKG_CONFIG_LIBDIR internally
   SubBanner "Move pkgconfig scripts"
   mkdir -p ${INSTALL_ROOT}/usr/lib/pkgconfig
   mv ${INSTALL_ROOT}/usr/lib/${arch}-${os}/pkgconfig/* \
       ${INSTALL_ROOT}/usr/lib/pkgconfig
+
+  # Temporary workaround for invalid implicit conversion from void* in pipewire.
+  # This is already fixed upstream in [1], so this can be removed once it rolls
+  # into Debian.
+  # [1] https://github.com/PipeWire/pipewire/commit/371da358d1580dc06218d18a12a99611cac39e4e
+  local pipewire_utils_h="${INSTALL_ROOT}/usr/include/pipewire/utils.h"
+  sed -i 's/malloc/(struct spa_pod*)malloc/' "${pipewire_utils_h}"
 }
 
 

build/linux/sysroot_scripts/sysroots.json

 {
     "sid_amd64": {
-        "Sha1Sum": "e7c53f04bd88d29d075bfd1f62b073aeb69cbe09",
+        "Sha1Sum": "bcc994cc6e5d4d6f0eec8b44e7f0a65f5a1a7b90",
         "SysrootDir": "debian_sid_amd64-sysroot",
         "Tarball": "debian_sid_amd64_sysroot.tar.xz"
     },
     "sid_arm": {
-        "Sha1Sum": "ef5c4f84bcafb7a3796d36bb1db7826317dde51c",
+        "Sha1Sum": "76e6068f9f6954e2ab1ff98ce5fa236d3d85bcbd",
         "SysrootDir": "debian_sid_arm-sysroot",
         "Tarball": "debian_sid_arm_sysroot.tar.xz"
     },
     "sid_arm64": {
-        "Sha1Sum": "953c2471bc7e71a788309f6c2d2003e8b703305d",
+        "Sha1Sum": "1b47b4930e197ab92d67c5fad1246cf27fa3b0ab",
         "SysrootDir": "debian_sid_arm64-sysroot",
         "Tarball": "debian_sid_arm64_sysroot.tar.xz"
     },
     "sid_armel": {
-        "Sha1Sum": "07a9da7e88e8b6fa550b053e4338c2caa4db15f0",
+        "Sha1Sum": "6f2b39bdf73db9d378c5309f4a7ec050b97e771e",
         "SysrootDir": "debian_sid_armel-sysroot",
         "Tarball": "debian_sid_armel_sysroot.tar.xz"
     },
     "sid_i386": {
-        "Sha1Sum": "9e6279438ece6fb42b5333ca90d5e9d0c188a403",
+        "Sha1Sum": "632bb77cc527cc90b94a302f550d99e99baaab87",
         "SysrootDir": "debian_sid_i386-sysroot",
         "Tarball": "debian_sid_i386_sysroot.tar.xz"
     },
     "sid_mips": {
-        "Sha1Sum": "958731a68a169631c0450efb15410ccc4135ef2a",
+        "Sha1Sum": "8d58231734ef581d33e3ff620cfa3d6ff7a7d964",
         "SysrootDir": "debian_sid_mips-sysroot",
         "Tarball": "debian_sid_mips_sysroot.tar.xz"
     },
     "sid_mips64el": {
-        "Sha1Sum": "51ca1f4092ac76ad1a1da953f0f3ce1aea947a42",
+        "Sha1Sum": "2a44ca6e7e1f442ed2f29c84805637c278cbe2a2",
         "SysrootDir": "debian_sid_mips64el-sysroot",
         "Tarball": "debian_sid_mips64el_sysroot.tar.xz"
     }