Reland "Android: Use locally-build devil deps only when build_with_chromium=true"

This reverts commit b47e9c83.

Reason for reland: Updated more .pydeps files.

Original change's description:
> Revert "Android: Use locally-build devil deps only when build_with_chromium=true"
>
> This reverts commit 49fdeca1.
>
> Reason for revert: Breaking presubmit due to stale .pydeps file:
> android_webview/tools/run_cts.pydeps
>
> Original change's description:
> > Android: Use locally-build devil deps only when build_with_chromium=true
> >
> > Bug: 1120190
> > Change-Id: I763e42a7ae69d14eb61caebd220844b756d6d171
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2480842
> > Commit-Queue: Andrew Grieve <agrieve@chromium.org>
> > Reviewed-by: Ben Pastene <bpastene@chromium.org>
> > Cr-Commit-Position: refs/heads/master@{#819489}
>
> TBR=agrieve@chromium.org,bpastene@chromium.org
>
> Change-Id: I960ad355283c66951e5931f61e25421c5a5dfdf7
> No-Presubmit: true
> No-Tree-Checks: true
> No-Try: true
> Bug: 1120190
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2490857
> Reviewed-by: Andrew Grieve <agrieve@chromium.org>
> Commit-Queue: Andrew Grieve <agrieve@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#819668}

TBR=agrieve # reland

Bug: 1120190
Change-Id: I8f3e21182e83a3e88da52331d56a8bef76507939
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2490558
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#819810}

BUILD.gn

@@ -1253,7 +1253,11 @@ if (!is_ios) {
                          root_build_dir) + ")" ]
 
     data = [
+      # These tests use //build/android/devil_chromium.py even when !is_android,
+      # so cannot use the helpers in //build/android (they assert(is_android)).
       "//build/android/",
+      "//build/gn_helpers.py",
+      "//build/config/gclient_args.gni",
       "//components/crash/content/tools/generate_breakpad_symbols.py",
       "//third_party/blink/renderer/bindings/scripts/",
       "//third_party/blink/renderer/build/scripts/",

android_webview/test/BUILD.gn

@@ -24,7 +24,6 @@ python_library("webview_cts_tests") {
   pydeps_file = "//android_webview/tools/run_cts.pydeps"
   deps = [ "//android_webview:system_webview_apk" ]
   data_deps = [
-    "//build/android:logdog_wrapper_py",
     "//build/android:test_runner_py",
     "//testing/buildbot/filters:webview_cts_tests_filters",
   ]

android_webview/tools/run_cts.pydeps

@@ -13,6 +13,7 @@
 //build/android/pylib/local/emulator/proto/avd_pb2.py
 //build/android/pylib/utils/__init__.py
 //build/android/pylib/utils/test_filter.py
+//build/gn_helpers.py
 //third_party/catapult/common/py_utils/py_utils/__init__.py
 //third_party/catapult/common/py_utils/py_utils/cloud_storage.py
 //third_party/catapult/common/py_utils/py_utils/cloud_storage_global_lock.py

build/android/BUILD.gn

@@ -52,7 +52,25 @@ python_library("devil_chromium_py") {
     "devil_chromium.json",
     "//third_party/catapult/third_party/gsutil/",
     "//third_party/catapult/devil/devil/devil_dependencies.json",
+
+    # Read by gn_helpers.BuildWithChromium()
+    "//build/config/gclient_args.gni",
+  ]
+}
+
+# Contains runtime deps for installing apks.
+# E.g. from test_runner.py or from apk_operations.py.
+group("apk_installer_data") {
+  # Other //build users let devil library fetch these from Google Storage.
+  if (build_with_chromium) {
+    data_deps = [
+      "//build/android/pylib/device/commands",
+      "//tools/android/md5sum",
     ]
+    data = [
+      "//third_party/android_build_tools/bundletool/bundletool-all-1.2.0.jar",
+    ]
+  }
 }
 
 python_library("test_runner_py") {
@@ -69,14 +87,17 @@ python_library("test_runner_py") {
     "${android_sdk_root}/platform-tools/adb",
     "//third_party/requests/",
   ]
-  data_deps = [ ":devil_chromium_py" ]
-  if (build_with_chromium) {
-    data += [
-      "//third_party/android_build_tools/bundletool/bundletool-all-1.2.0.jar",
-      "//tools/android/avd/proto/",
+  data_deps = [
+    ":apk_installer_data",
+    ":devil_chromium_py",
+    ":logdog_wrapper_py",
+    ":stack_tools",
   ]
-    data_deps +=
-        [ "//third_party/android_platform/development/scripts:stack_py" ]
+
+  # Other //build users let devil library fetch these from Google Storage.
+  if (build_with_chromium) {
+    data_deps += [ "//tools/android/forwarder2" ]
+    data += [ "//tools/android/avd/proto/" ]
     if (is_asan) {
       data_deps += [ "//tools/android/asan/third_party:asan_device_setup" ]
     }
@@ -104,19 +125,14 @@ python_library("resource_sizes_py") {
   ]
 }
 
-python_library("bundle_wrapper_script_py") {
-  pydeps_file = "gyp/create_bundle_wrapper_script.pydeps"
-  data = [
-    "//third_party/android_build_tools/bundletool/bundletool-all-1.2.0.jar",
-  ]
-}
-
 # Tools necessary for symbolizing tombstones or stack traces that are output to
 # logcat.
 # Hidden behind build_with_chromium because some third party repos that use
 # //build don't pull in //third_party/android_platform.
-if (build_with_chromium) {
-  group("stack_tools") {
+# TODO(crbug.com/1120190): Move stack script into //build/third_party
+#     and enable unconditionally.
+group("stack_tools") {
+  if (build_with_chromium) {
     data = [
       "tombstones.py",
       "pylib/symbols/",

build/android/devil_chromium.py

@@ -7,14 +7,21 @@
 import os
 import sys
 
+from pylib import constants
 from pylib.constants import host_paths
 
 if host_paths.DEVIL_PATH not in sys.path:
-  sys.path.append(host_paths.DEVIL_PATH)
+  sys.path.insert(1, host_paths.DEVIL_PATH)
 
 from devil import devil_env
 from devil.android.ndk import abis
 
+_BUILD_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'build')
+if _BUILD_DIR not in sys.path:
+  sys.path.insert(1, _BUILD_DIR)
+
+import gn_helpers
+
 _DEVIL_CONFIG = os.path.abspath(
     os.path.join(os.path.dirname(__file__), 'devil_chromium.json'))
 
@@ -107,6 +114,33 @@ _DEVIL_BUILD_PRODUCT_DEPS = {
 }
 
 
+def _UseLocalBuildProducts(output_directory, devil_dynamic_config):
+  output_directory = os.path.abspath(output_directory)
+  devil_dynamic_config['dependencies'] = {
+      dep_name: {
+          'file_info': {
+              '%s_%s' % (dep_config['platform'], dep_config['arch']): {
+                  'local_paths': [
+                      os.path.join(output_directory,
+                                   *dep_config['path_components']),
+                  ],
+              }
+              for dep_config in dep_configs
+          }
+      }
+      for dep_name, dep_configs in _DEVIL_BUILD_PRODUCT_DEPS.iteritems()
+  }
+
+
+def _BuildWithChromium():
+  """Returns value of gclient's |build_with_chromium|."""
+  gni_path = os.path.join(_BUILD_DIR, 'config', 'gclient_args.gni')
+  with open(gni_path) as f:
+    data = f.read()
+  args = gn_helpers.FromGNArgs(data)
+  return args['build_with_chromium']
+
+
 def Initialize(output_directory=None, custom_deps=None, adb_path=None):
   """Initializes devil with chromium's binaries and third-party libraries.
 
@@ -134,26 +168,18 @@ def Initialize(output_directory=None, custom_deps=None, adb_path=None):
     adb_path: An optional path to use for the adb binary. If not set, this uses
       the adb binary provided by the Android SDK.
   """
+  build_with_chromium = _BuildWithChromium()
 
   devil_dynamic_config = {
     'config_type': 'BaseConfig',
     'dependencies': {},
   }
-  if output_directory:
-    output_directory = os.path.abspath(output_directory)
-    devil_dynamic_config['dependencies'] = {
-      dep_name: {
-        'file_info': {
-          '%s_%s' % (dep_config['platform'], dep_config['arch']): {
-            'local_paths': [
-              os.path.join(output_directory, *dep_config['path_components']),
-            ],
-          }
-          for dep_config in dep_configs
-        }
-      }
-      for dep_name, dep_configs in _DEVIL_BUILD_PRODUCT_DEPS.iteritems()
-    }
+  if build_with_chromium and output_directory:
+    # Non-chromium users of chromium's //build directory fetch build products
+    # from google storage rather than use locally built copies. Chromium uses
+    # locally-built copies so that changes to the tools can be easily tested.
+    _UseLocalBuildProducts(output_directory, devil_dynamic_config)
+
   if custom_deps:
     devil_dynamic_config['dependencies'].update(custom_deps)
   if adb_path:
@@ -167,5 +193,6 @@ def Initialize(output_directory=None, custom_deps=None, adb_path=None):
       }
     })
 
-  devil_env.config.Initialize(
-      configs=[devil_dynamic_config], config_files=[_DEVIL_CONFIG])
+  config_files = [_DEVIL_CONFIG] if build_with_chromium else None
+  devil_env.config.Initialize(configs=[devil_dynamic_config],
+                              config_files=config_files)

build/android/devil_chromium.pydeps

@@ -32,6 +32,7 @@
 ../../third_party/catapult/devil/devil/utils/timeout_retry.py
 ../../third_party/catapult/devil/devil/utils/watchdog_timer.py
 ../../third_party/catapult/third_party/zipfile/zipfile_2_7_13.py
+../gn_helpers.py
 devil_chromium.py
 pylib/__init__.py
 pylib/constants/__init__.py

build/config/android/internal_rules.gni

@@ -700,7 +700,7 @@ template("test_runner_script") {
                                "deps",
                                "public_deps",
                              ])
-      data_deps = [ "//tools/android/md5sum" ]
+      data_deps = []
       if (defined(invoker.data_deps)) {
         data_deps += invoker.data_deps
       }
@@ -726,18 +726,6 @@ template("test_runner_script") {
 
   generate_android_wrapper(target_name) {
     wrapper_script = "$root_build_dir/bin/run_${_test_name}"
-    forward_variables_from(invoker,
-                           [
-                             "data_deps",
-                             "deps",
-                           ])
-    if (!defined(deps)) {
-      deps = []
-    }
-
-    if (!defined(data_deps)) {
-      data_deps = []
-    }
 
     if (defined(android_test_runner_script)) {
       executable = android_test_runner_script
@@ -749,21 +737,20 @@ template("test_runner_script") {
     if (defined(invoker.deps)) {
       deps = invoker.deps
     }
-    data_deps = []
+    data_deps = [ "//build/android:test_runner_py" ]
     if (defined(invoker.data_deps)) {
-      data_deps = invoker.data_deps
+      data_deps += invoker.data_deps
     }
     data = []
+    if (defined(invoker.data)) {
+      data = invoker.data
+    }
 
     executable_args = [
       _test_type,
       "--output-directory",
       "@WrappedPath(.)",
     ]
-    data_deps += [
-      "//build/android:logdog_wrapper_py",
-      "//build/android:test_runner_py",
-    ]
 
     if (_runtime_deps) {
       deps += [ ":$_runtime_deps_target" ]

build/config/android/rules.gni

@@ -3304,7 +3304,6 @@ if (enable_java_templates) {
         _generated_script = "$root_build_dir/bin/${invoker.target_name}"
         script = "//build/android/gyp/create_apk_operations_script.py"
         outputs = [ _generated_script ]
-        data_deps = [ "//tools/android/md5sum" ]
         args = [
           "--script-output-path",
           rebase_path(_generated_script, root_build_dir),
@@ -3338,9 +3337,12 @@ if (enable_java_templates) {
             _static_library_apk_path,
           ]
         }
-        if (!defined(data)) {
         data = []
-        }
+        data_deps = [
+          "//build/android:apk_installer_data",
+          "//build/android:stack_tools",
+        ]
+
         if (_proguard_enabled && !_incremental_apk) {
           # Required by logcat command.
           data_deps += [ "//build/android/stacktrace:java_deobfuscate" ]
@@ -3407,10 +3409,6 @@ if (enable_java_templates) {
         data_deps += [ ":${target_name}__lint" ]
       }
 
-      if (_incremental_apk) {
-        # device/commands is used by the installer script to push files via .zip.
-        data_deps += [ "//build/android/pylib/device/commands" ]
-      }
       if (_uses_static_library) {
         data_deps += [ invoker.static_library_provider ]
       }
@@ -3778,8 +3776,6 @@ if (enable_java_templates) {
         # symbolization can be done.
         ":${target_name}__secondary_abi_shared_library_list",
         ":${target_name}__shared_library_list",
-        "//build/android/pylib/device/commands",
-        "//tools/android/forwarder2",
       ]
       if (defined(invoker.data_deps)) {
         data_deps += invoker.data_deps
@@ -3957,10 +3953,6 @@ if (enable_java_templates) {
         "//base:base_java",
         "//testing/android/reporter:reporter_java",
       ]
-      data_deps += [ "//build/android/pylib/device/commands" ]
-      if (host_os == "linux") {
-        data_deps += [ "//tools/android/forwarder2" ]
-      }
     }
   }
 
@@ -4990,8 +4982,8 @@ if (enable_java_templates) {
         _bundle_path,
       ]
       data_deps = [
-        "//build/android:bundle_wrapper_script_py",
-        "//tools/android/md5sum",
+        "//build/android:apk_installer_data",
+        "//build/android:stack_tools",
       ]
 
       deps = [ _base_module_build_config_target ]

chrome/android/monochrome/scripts/monochrome_python_tests.pydeps

 # Generated by running:
-#   build/print_python_deps.py --gn-paths chrome/android/monochrome/scripts
+#   build/print_python_deps.py --output chrome/android/monochrome/scripts/monochrome_python_tests.pydeps --gn-paths chrome/android/monochrome/scripts
 //build/android/devil_chromium.py
 //build/android/pylib/__init__.py
 //build/android/pylib/constants/__init__.py
 //build/android/pylib/constants/host_paths.py
+//build/gn_helpers.py
 //chrome/android/monochrome/scripts/monochrome_android_manifest_test.py
 //chrome/android/monochrome/scripts/monochrome_apk_checker_test.py
 //chrome/android/monochrome/scripts/monochrome_dexdump_test.py

chrome/test/chromedriver/test/run_py_tests.pydeps

@@ -4,6 +4,7 @@
 ../../../../build/android/pylib/__init__.py
 ../../../../build/android/pylib/constants/__init__.py
 ../../../../build/android/pylib/constants/host_paths.py
+../../../../build/gn_helpers.py
 ../../../../third_party/catapult/common/py_utils/py_utils/__init__.py
 ../../../../third_party/catapult/common/py_utils/py_utils/cloud_storage.py
 ../../../../third_party/catapult/common/py_utils/py_utils/cloud_storage_global_lock.py

testing/scripts/run_android_wpt.pydeps

@@ -4,6 +4,7 @@
 //build/android/pylib/__init__.py
 //build/android/pylib/constants/__init__.py
 //build/android/pylib/constants/host_paths.py
+//build/gn_helpers.py
 //testing/scripts/common.py
 //testing/scripts/run_android_wpt.py
 //testing/scripts/wpt_common.py