Commit b8b65994 authored by Brian Sheedy's avatar Brian Sheedy Committed by Commit Bot

Apply GPU BQ export to all builders

Applies the ResultDB BigQuery export for the GPU-specific test result
tables to all builders instead of only specific ones. This is to catch
cases where GPU tests are run on non-GPU builders, whose results we
still want to capture.

This results in ~3.5x more CI and ~3x more try builders potentially
exporting to the BQ table, although the vast majority of those
additions won't actually have any results that match the test ID regex.

Also adjusts the test ID regex to also match the
fuchsia_telemetry_gpu_integration_test target, which is effectively the
same as the existing telemetry_gpu_integration_test that the current
regex matches, but for the Fuchsia platform.

Bug: 1134751
Change-Id: I4f2dbfc96724b62229e04490b89b30388af799be
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2518139
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Reviewed-by: default avatarChan Li <chanli@chromium.org>
Reviewed-by: default avatarNodir Turakulov <nodir@chromium.org>
Reviewed-by: default avatarGarrett Beaty <gbeaty@chromium.org>
Cr-Commit-Position: refs/heads/master@{#825921}
parent 8689534d
This diff is collapsed.
...@@ -375,8 +375,10 @@ def ci_builder( ...@@ -375,8 +375,10 @@ def ci_builder(
notifies - Any extra notifiers to attach to this builder. notifies - Any extra notifiers to attach to this builder.
resultdb_bigquery_exports - a list of resultdb.export_test_results(...) resultdb_bigquery_exports - a list of resultdb.export_test_results(...)
specifying additional parameters for exporting test results to BigQuery. specifying additional parameters for exporting test results to BigQuery.
Will always upload to the luci-resultdb.chromium.ci_test_results table Will always upload to the following tables in addition to any tables
in addition to any tables specified by the list's elements. specified by the list's elements:
luci-resultdb.chromium.ci_test_results
luci-resultdb.chromium.gpu_ci_test_results
experiments - a dict of experiment name to the percentage chance (0-100) experiments - a dict of experiment name to the percentage chance (0-100)
that it will apply to builds generated from this builder. that it will apply to builds generated from this builder.
""" """
...@@ -393,6 +395,14 @@ def ci_builder( ...@@ -393,6 +395,14 @@ def ci_builder(
resultdb.export_test_results( resultdb.export_test_results(
bq_table = "luci-resultdb.chromium.ci_test_results", bq_table = "luci-resultdb.chromium.ci_test_results",
), ),
resultdb.export_test_results(
bq_table = "luci-resultdb.chromium.gpu_ci_test_results",
predicate = resultdb.test_result_predicate(
# Only match the telemetry_gpu_integration_test and
# fuchsia_telemetry_gpu_integration_test targets.
test_id_regexp = "ninja://(chrome/test:|content/test:fuchsia_)telemetry_gpu_integration_test/.+",
),
),
] ]
merged_resultdb_bigquery_exports.extend(resultdb_bigquery_exports or []) merged_resultdb_bigquery_exports.extend(resultdb_bigquery_exports or [])
...@@ -704,15 +714,6 @@ def fyi_windows_builder( ...@@ -704,15 +714,6 @@ def fyi_windows_builder(
**kwargs **kwargs
) )
gpu_ci_resultdb_exports = [
resultdb.export_test_results(
bq_table = "luci-resultdb.chromium.gpu_ci_test_results",
predicate = resultdb.test_result_predicate(
test_id_regexp = "ninja://chrome/test:telemetry_gpu_integration_test/.+",
),
),
]
def gpu_fyi_builder(*, name, **kwargs): def gpu_fyi_builder(*, name, **kwargs):
return ci.builder( return ci.builder(
name = name, name = name,
...@@ -722,7 +723,6 @@ def gpu_fyi_builder(*, name, **kwargs): ...@@ -722,7 +723,6 @@ def gpu_fyi_builder(*, name, **kwargs):
properties = { properties = {
"perf_dashboard_machine_group": "ChromiumGPUFYI", "perf_dashboard_machine_group": "ChromiumGPUFYI",
}, },
resultdb_bigquery_exports = gpu_ci_resultdb_exports,
**kwargs **kwargs
) )
...@@ -787,7 +787,6 @@ def gpu_builder(*, name, tree_closing = True, notifies = None, **kwargs): ...@@ -787,7 +787,6 @@ def gpu_builder(*, name, tree_closing = True, notifies = None, **kwargs):
builder_group = "chromium.gpu", builder_group = "chromium.gpu",
tree_closing = tree_closing, tree_closing = tree_closing,
notifies = notifies, notifies = notifies,
resultdb_bigquery_exports = gpu_ci_resultdb_exports,
**kwargs **kwargs
) )
......
...@@ -155,8 +155,10 @@ def try_builder( ...@@ -155,8 +155,10 @@ def try_builder(
that it will apply to builds generated from this builder. that it will apply to builds generated from this builder.
resultdb_bigquery_exports - a list of resultdb.export_test_results(...) resultdb_bigquery_exports - a list of resultdb.export_test_results(...)
specifying additional parameters for exporting test results to BigQuery. specifying additional parameters for exporting test results to BigQuery.
Will always upload to the luci-resultdb.chromium.try_test_results table Will always upload to the following tables in addition to any tables
in addition to any tables specified by the list's elements. specified by the list's elements:
luci-resultdb.chromium.try_test_results
luci-resultdb.chromium.gpu_try_test_results
""" """
if not branches.matches(branch_selector): if not branches.matches(branch_selector):
return return
...@@ -169,6 +171,14 @@ def try_builder( ...@@ -169,6 +171,14 @@ def try_builder(
resultdb.export_test_results( resultdb.export_test_results(
bq_table = "luci-resultdb.chromium.try_test_results", bq_table = "luci-resultdb.chromium.try_test_results",
), ),
resultdb.export_test_results(
bq_table = "luci-resultdb.chromium.gpu_try_test_results",
predicate = resultdb.test_result_predicate(
# Only match the telemetry_gpu_integration_test and
# fuchsia_telemetry_gpu_integration_test targets.
test_id_regexp = "ninja://(chrome/test:|content/test:fuchsia_)telemetry_gpu_integration_test/.+",
),
),
] ]
merged_resultdb_bigquery_exports.extend( merged_resultdb_bigquery_exports.extend(
defaults.get_value( defaults.get_value(
...@@ -416,22 +426,12 @@ def chromium_win_builder( ...@@ -416,22 +426,12 @@ def chromium_win_builder(
**kwargs **kwargs
) )
gpu_try_resultdb_exports = [
resultdb.export_test_results(
bq_table = "luci-resultdb.chromium.gpu_try_test_results",
predicate = resultdb.test_result_predicate(
test_id_regexp = "ninja://chrome/test:telemetry_gpu_integration_test/.+",
),
),
]
def gpu_try_builder(*, name, builderless = False, execution_timeout = 6 * time.hour, **kwargs): def gpu_try_builder(*, name, builderless = False, execution_timeout = 6 * time.hour, **kwargs):
return try_builder( return try_builder(
name = name, name = name,
builderless = builderless, builderless = builderless,
execution_timeout = execution_timeout, execution_timeout = execution_timeout,
service_account = "chromium-try-gpu-builder@chops-service-accounts.iam.gserviceaccount.com", service_account = "chromium-try-gpu-builder@chops-service-accounts.iam.gserviceaccount.com",
resultdb_bigquery_exports = gpu_try_resultdb_exports,
**kwargs **kwargs
) )
...@@ -500,5 +500,4 @@ try_ = struct( ...@@ -500,5 +500,4 @@ try_ = struct(
gpu_chromium_linux_builder = gpu_chromium_linux_builder, gpu_chromium_linux_builder = gpu_chromium_linux_builder,
gpu_chromium_mac_builder = gpu_chromium_mac_builder, gpu_chromium_mac_builder = gpu_chromium_mac_builder,
gpu_chromium_win_builder = gpu_chromium_win_builder, gpu_chromium_win_builder = gpu_chromium_win_builder,
gpu_try_resultdb_exports = gpu_try_resultdb_exports,
) )
...@@ -30,7 +30,6 @@ try_.defaults.set( ...@@ -30,7 +30,6 @@ try_.defaults.set(
subproject_list_view = "luci.chromium.try", subproject_list_view = "luci.chromium.try",
swarming_tags = ["vpython:native-python-wrapper"], swarming_tags = ["vpython:native-python-wrapper"],
task_template_canary_percentage = 5, task_template_canary_percentage = 5,
resultdb_bigquery_exports = try_.gpu_try_resultdb_exports,
) )
# Builders appear after the function used to define them, with all builders # Builders appear after the function used to define them, with all builders
......
...@@ -336,8 +336,6 @@ try_.chromium_android_builder( ...@@ -336,8 +336,6 @@ try_.chromium_android_builder(
ssd = True, ssd = True,
use_java_coverage = True, use_java_coverage = True,
tryjob = try_.job(), tryjob = try_.job(),
# Due to mirroring chromium.gpu's "Android Release (Nexus 5X)".
resultdb_bigquery_exports = try_.gpu_try_resultdb_exports,
) )
try_.chromium_android_builder( try_.chromium_android_builder(
...@@ -936,8 +934,6 @@ try_.chromium_linux_builder( ...@@ -936,8 +934,6 @@ try_.chromium_linux_builder(
try_.chromium_linux_builder( try_.chromium_linux_builder(
name = "linux-dcheck-off-rel", name = "linux-dcheck-off-rel",
# Due to mirroring chromium.gpu's "Linux Release (NVIDIA)".
resultdb_bigquery_exports = try_.gpu_try_resultdb_exports,
) )
try_.chromium_linux_builder( try_.chromium_linux_builder(
...@@ -997,8 +993,6 @@ try_.chromium_linux_builder( ...@@ -997,8 +993,6 @@ try_.chromium_linux_builder(
main_list_view = "try", main_list_view = "try",
tryjob = try_.job(), tryjob = try_.job(),
use_clang_coverage = True, use_clang_coverage = True,
# Due to mirroring chromium.gpu's "Linux Release (NVIDIA)".
resultdb_bigquery_exports = try_.gpu_try_resultdb_exports,
) )
try_.chromium_linux_builder( try_.chromium_linux_builder(
...@@ -1007,8 +1001,6 @@ try_.chromium_linux_builder( ...@@ -1007,8 +1001,6 @@ try_.chromium_linux_builder(
goma_jobs = goma.jobs.J150, goma_jobs = goma.jobs.J150,
tryjob = try_.job(experiment_percentage = 10), tryjob = try_.job(experiment_percentage = 10),
use_clang_coverage = True, use_clang_coverage = True,
# Due to mirroring chromium.gpu's "Linux Release (NVIDIA)".
resultdb_bigquery_exports = try_.gpu_try_resultdb_exports,
) )
try_.chromium_linux_builder( try_.chromium_linux_builder(
...@@ -1224,9 +1216,6 @@ try_.chromium_mac_builder( ...@@ -1224,9 +1216,6 @@ try_.chromium_mac_builder(
main_list_view = "try", main_list_view = "try",
os = os.MAC_DEFAULT, os = os.MAC_DEFAULT,
tryjob = try_.job(), tryjob = try_.job(),
# Due to mirroring chromium.gpu's "Mac Release (Intel)" and
# "Mac Retina Release (AMD)".
resultdb_bigquery_exports = try_.gpu_try_resultdb_exports,
) )
try_.chromium_mac_builder( try_.chromium_mac_builder(
...@@ -1497,8 +1486,6 @@ try_.chromium_win_builder( ...@@ -1497,8 +1486,6 @@ try_.chromium_win_builder(
# of stale tryjobs gracefully, resulting in longer average build times. # of stale tryjobs gracefully, resulting in longer average build times.
# Windows is particularly susceptible to this. # Windows is particularly susceptible to this.
tryjob = try_.job(cancel_stale = False), tryjob = try_.job(cancel_stale = False),
# Due to mirroring chromium.gpu's "Win10 x64 Release (NVIDIA)".
resultdb_bigquery_exports = try_.gpu_try_resultdb_exports,
) )
try_.chromium_win_builder( try_.chromium_win_builder(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment