Reland "Ignore weak undefined symbols when calculating dpkg dependencies"

This is a reland of 5f865cce

Original change's description:
> Ignore weak undefined symbols when calculating dpkg dependencies
> 
> This is needed after [1] used weak undefined symbols to test feature
> support in libgbm. The dependencies for the Chrome debian package are
> unchanged before/after this change.
> 
> [1] https://chromium-review.googlesource.com/c/chromium/src/+/1988382
> 
> BUG=1031269
> R=thestig,thakis
> CC=spang
> 
> Change-Id: I760033d5883bf65a5a39316ea382766ae3754088
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1992340
> Reviewed-by: Nico Weber <thakis@chromium.org>
> Reviewed-by: Lei Zhang <thestig@chromium.org>
> Commit-Queue: Thomas Anderson <thomasanderson@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#730271}

Bug: 1031269
Change-Id: I1975969c780e5b7e2d9974526a8527a4f3d36143
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2001578
Auto-Submit: Thomas Anderson <thomasanderson@chromium.org>
Reviewed-by: Nico Weber <thakis@chromium.org>
Commit-Queue: Thomas Anderson <thomasanderson@chromium.org>
Cr-Commit-Position: refs/heads/master@{#732231}

chrome/installer/linux/BUILD.gn

@@ -76,6 +76,7 @@ action_foreach("calculate_deb_dependencies") {
     "debian/deb_version.py",
     "debian/dist_package_versions.json",
     "debian/package_version_interval.py",
+    "//third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl",
   ]
   outputs = [
     "$root_out_dir/deb_{{source_name_part}}.deps",

chrome/installer/linux/debian/calculate_package_deps.py

@@ -31,7 +31,11 @@ arch = args.arch
 dep_filename = os.path.abspath(args.dep_filename)
 distro_check = args.distro_check
 
-cmd = ['dpkg-shlibdeps']
+script_dir = os.path.dirname(os.path.realpath(__file__))
+dpkg_shlibdeps = os.path.join(script_dir, '..', '..', '..', '..', 'third_party',
+                              'dpkg-shlibdeps', 'dpkg-shlibdeps.pl')
+
+cmd = [dpkg_shlibdeps, '--ignore-weak-undefined']
 if arch == 'x64':
   cmd.extend(['-l%s/usr/lib/x86_64-linux-gnu' % sysroot,
               '-l%s/lib/x86_64-linux-gnu' % sysroot])
@@ -64,7 +68,7 @@ if exit_code != 0:
   print 'stderr was ' + stderr
   sys.exit(1)
 
-deps_str = stdout.replace('shlibs:Depends=', '').replace('\n', '')
+deps_str = re.sub(r'.*shlibs\:Depends\=', '', stdout.replace('\n', ''))
 deps = deps_str.split(', ')
 interval_sets = []
 if deps_str != '':

third_party/dpkg-shlibdeps/METADATA

+name: "dpkg"
+description:
+    "package manager for Debian"
+
+third_party {
+  url {
+    type: HOMEPAGE
+    value: "https://wiki.debian.org/Teams/Dpkg"
+  }
+  url {
+    type: GIT
+    value: "https://git.dpkg.org/git/dpkg/dpkg.git"
+  }
+  version: "a0828e04289639f875cb924f8d1a65a634683b19"
+  last_upgrade_date { year: 2020 month: 1 day: 9 }
+  license_type: RESTRICTED
+}

third_party/dpkg-shlibdeps/OWNERS

+thestig@chromium.org
+thomasanderson@chromium.org
+
+# COMPONENT: Tools

third_party/dpkg-shlibdeps/README.chromium

+Name: dpkg-shlibdeps
+URL: http://anonscm.debian.org/cgit/dpkg/dpkg.git
+Version: a0828e04289639f875cb924f8d1a65a634683b19
+License: GPL v2
+License File: NOT_SHIPPED
+Security Critical: no
+
+Description:
+Generate shared library substvar dependencies. This directory can be removed
+if/when the --ignore-weak-undefined option becomes available on the buildbots.
+
+Updating:
+Copy dpkg/COPYING and dpkg/scripts/dpkg-shlibdeps.pl to
+third_party/dpkg-shlibdeps/.
+
+Patches:
+- 0001-dkpg-shlibdeps-add-option-to-ignore-weak-undefined-symbols.patch
+  Prevents weak undefined symbols from introducing hard dependencies. Upstream
+  merge request:
+  https://salsa.debian.org/thomasanderson-guest/dpkg/merge_requests/1
+- 0002-fix-compatibility-for-chromium.patch
+  dpkg-shlibdeps.pl comes from dpkg-dev, which depends on libdpkg-perl that
+  provides perl modules for eg. locating shared libraries and getting a list of
+  symbols from them. The API of libdpkg-perl is not stable (but changes very
+  infrequently), so in some cases dpkg-shlibdeps.pl will be expecting APIs that
+  do not exist on the system. This patch is a minimal set of changes that fixes
+  compatibility for supported build environments (Ubuntu Xenial and later).
+  Specifically, provide the debug() function, and don't expect exec_abi to be a
+  member of Dpkg::Shlibs::Objdump::Object.

third_party/dpkg-shlibdeps/patches/0001-dkpg-shlibdeps-add-option-to-ignore-weak-undefined-symbols.patch

+From 59477dae6f2c68f0ebd9ccca18e737b67408bc11 Mon Sep 17 00:00:00 2001
+From: Tom Anderson <thomasanderson@chromium.org>
+Date: Wed, 8 Jan 2020 10:55:20 -0800
+Subject: [PATCH 1/2] dkpg-shlibdeps: Add option to ignore weak undefined
+ symbols
+
+Weak undefined symbols can be used to test feature support. For example:
+
+__attribute__((weak)) uint64_t gbm_bo_get_modifier(struct gbm_bo* bo);
+void foo(gbm_bo* bo) {
+    // gbm_create_device is a strong undefined symbol. The dynamic linker will
+    // fail if this is not available.
+    gbm_create_device(1234);
+    // gbm_bo_get_modifier is a weak undefined symbol. It will be NULL if it's
+    // not available.
+    if (gbm_bo_get_modifier)
+        gbm_bo_get_modifier(bo);
+}
+
+However currently, dpkg-shlibdeps would consider gbm_bo_get_modifier in the
+example to be a required symbol (and would increase the minimum required library
+version), even though the intention is to use it only if it is available.
+
+To address this, this change skips weak undefined symbols when checking for
+dependencies. Out of an abundance of caution, this behavior is guarded behind a
+new flag: --ignore-weak-undefined.
+---
+ scripts/dpkg-shlibdeps.pl | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/scripts/dpkg-shlibdeps.pl b/scripts/dpkg-shlibdeps.pl
+index 8142c85b4..e3d8c7200 100755
+--- a/scripts/dpkg-shlibdeps.pl
++++ b/scripts/dpkg-shlibdeps.pl
+@@ -67,6 +67,7 @@ my $varlistfile = 'debian/substvars';
+ my $varlistfilenew;
+ my $varnameprefix = 'shlibs';
+ my $ignore_missing_info = 0;
++my $ignore_weak_undefined = 0;
+ my $warnings = WARN_SYM_NOT_FOUND | WARN_DEP_AVOIDABLE;
+ my $debug = 0;
+ my @exclude = ();
+@@ -120,6 +121,8 @@ foreach (@ARGV) {
+ 	}
+     } elsif (m/^--ignore-missing-info$/) {
+ 	$ignore_missing_info = 1;
++    } elsif (m/^--ignore-weak-undefined$/) {
++	$ignore_weak_undefined = 1;
+     } elsif (m/^--warnings=(\d+)$/) {
+ 	$warnings = $1;
+     } elsif (m/^-t(.*)$/) {
+@@ -365,6 +368,7 @@ foreach my $file (keys %exec) {
+     }
+     debug(2, 'Analyzing all undefined symbols');
+     foreach my $sym ($obj->get_undefined_dynamic_symbols()) {
++	next if $ignore_weak_undefined && $sym->{weak};
+ 	my $name = $sym->{name};
+ 	if ($sym->{version}) {
+ 	    $name .= '@' . "$sym->{version}";
+@@ -610,6 +614,8 @@ sub usage {
+                              in the given build directory.
+   -v                       enable verbose mode (can be used multiple times).
+   --ignore-missing-info    don't fail if dependency information can't be found.
++  --ignore-weak-undefined  only allow strong undefined symbols to introduce
++                             dependencies.
+   --warnings=<value>       define set of active warnings (see manual page).
+   --admindir=<directory>   change the administrative directory.
+   -?, --help               show this help message.
+-- 
+2.25.0.rc1.283.g88dfdc4193-goog
+

third_party/dpkg-shlibdeps/patches/0002-fix-compatibility-for-chromium.patch

+From 3f179bf13081f9dde0535a17864ceeba685ab907 Mon Sep 17 00:00:00 2001
+From: Tom Anderson <thomasanderson@chromium.org>
+Date: Wed, 8 Jan 2020 12:11:56 -0800
+Subject: [PATCH 2/2] Fix compatibility for Chromium
+
+* Add debug() and g_() functions in case the system dpkg doesn't have it.
+* Implement List::Util::any and List::Util::none for older versions of perl.
+---
+ scripts/dpkg-shlibdeps.pl | 20 ++++++++++++++------
+ 1 file changed, 14 insertions(+), 6 deletions(-)
+
+diff --git a/scripts/dpkg-shlibdeps.pl b/scripts/dpkg-shlibdeps.pl
+index e3d8c7200..654dfa898 100755
+--- a/scripts/dpkg-shlibdeps.pl
++++ b/scripts/dpkg-shlibdeps.pl
+@@ -21,11 +21,10 @@
+ # You should have received a copy of the GNU General Public License
+ # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ 
+-use strict;
+ use warnings;
+ use feature qw(state);
+ 
+-use List::Util qw(any none);
++use List::Util qw(reduce);
+ use Cwd qw(realpath);
+ use File::Basename qw(dirname);
+ 
+@@ -51,6 +50,11 @@ use constant {
+     WARN_NOT_NEEDED => 4,
+ };
+ 
++sub none (&@) { my $code=shift; reduce { $a && !$code->(local $_ = $b) } 1, @_; }
++sub any  (&@) { my $code=shift; reduce { $a ||  $code->(local $_ = $b) } 0, @_; }
++
++sub g_ { return shift; }
++
+ # By increasing importance
+ my @depfields = qw(Suggests Recommends Depends Pre-Depends);
+ my $i = 0; my %depstrength = map { $_ => $i++ } @depfields;
+@@ -153,6 +157,11 @@ sub ignore_pkgdir {
+     return any { $path =~ /^\Q$_\E/ } @pkg_dir_to_ignore;
+ }
+ 
++sub debug {
++    my $level = shift;
++    print @_ if $level <= $debug;
++}
++
+ if (-d 'debian') {
+     push @pkg_symbols, grep { !ignore_pkgdir($_) } glob 'debian/*/DEBIAN/symbols';
+     push @pkg_shlibs, grep { !ignore_pkgdir($_) } glob 'debian/*/DEBIAN/shlibs';
+@@ -198,18 +207,17 @@ foreach my $file (keys %exec) {
+     my %soname_notfound;
+     my %alt_soname;
+     foreach my $soname (@sonames) {
+-	my @libs = my_find_library($soname, $obj->{RPATH}, $obj->{exec_abi}, $file);
++	my @libs = my_find_library($soname, $obj->{RPATH}, $obj->{exec_abi} || $obj->{format}, $file);
+ 	unless (scalar @libs) {
+ 	    $soname_notfound{$soname} = 1;
+ 	    $global_soname_notfound{$soname} = 1;
+ 	    my $msg = g_('cannot find library %s needed by %s (ELF ' .
+ 	                 "format: '%s' abi: '%s'; RPATH: '%s')");
+-	    my $exec_abi = unpack 'H*', $obj->{exec_abi};
+ 	    if (scalar(split_soname($soname))) {
+-		errormsg($msg, $soname, $file, $obj->{format}, $exec_abi, join(':', @{$obj->{RPATH}}));
++		errormsg($msg, $soname, $file, $obj->{format}, join(':', @{$obj->{RPATH}}));
+ 		$error_count++;
+ 	    } else {
+-		warning($msg, $soname, $file, $obj->{format}, $exec_abi, join(':', @{$obj->{RPATH}}));
++		warning($msg, $soname, $file, $obj->{format}, join(':', @{$obj->{RPATH}}));
+ 	    }
+ 	    next;
+ 	}
+-- 
+2.25.0.rc1.283.g88dfdc4193-goog
+