Skip to content

GitLab

T
tangled
Commit 70ca76dc authored by Garrett Beaty's avatar Garrett Beaty Committed by Commit Bot
Browse files
Options

Add a presubmit to ensure our goma usage isn't modified during freeze. 

A generator is added that outputs some numbers representing our goma
usage. The values have no concrete meaning, but approximate the load
applied to goma for the project and the relative load for different
builders (with some numbers being very rough approximations).

Changes that modify the generated file require a footer in the CL
description in order to submit.

Bug: 1145230
Change-Id: I6d58deee27e9bc6d220f87311a6266c220f03e51
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2556210
Commit-Queue: Garrett Beaty <gbeaty@chromium.org>
Reviewed-by: default avatarErik Staab <estaab@chromium.org>
Cr-Commit-Position: refs/heads/master@{#831871}
parent 096c4b68
Hide whitespace changes
Inline Side-by-side
Showing with 917 additions and 2 deletions
infra/config/PRESUBMIT.py
View file @ 70ca76dc
......@@ -8,6 +8,9 @@ See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details on the presubmit API built into depot_tools.
"""
import ast
import traceback
PRESUBMIT_VERSION = '2.0.0'
......@@ -139,3 +142,35 @@ def CheckOutagesConfigOnCommit(input_api, output_api):
]
return []
_IGNORE_GOMA_FREEZE_FOOTER = 'Ignore-Goma-Freeze'
def CheckGomaUsageDuringFreeze(input_api, output_api):
for f in input_api.AffectedFiles():
if f.LocalPath() != 'infra/config/generated/goma-usage.pyl':
continue
if f.Action() == 'D':
break
def get_goma_usage(lines):
return ast.literal_eval('\n'.join(lines))
# Make sure that the file can be parsed
try:
new_usage = get_goma_usage(f.NewContents())
except Exception as e:
return [
output_api.PresubmitError('Failed to parse goma-usage.pyl',
long_text=traceback.format_exc()),
]
if f.Action() != 'M':
break
old_usage = get_goma_usage(f.OldContents())
if new_usage['*weighted total*'] > old_usage['*weighted total*']:
footers = input_api.change.GitFootersFromDescription()
if _IGNORE_GOMA_FREEZE_FOOTER not in footers:
return [output_api.PresubmitError('\n'.join([
('The goma team has a freeze on goma usage until January 5, 2021, '
'this change increases the usage of goma.'),
('If this change needs to be made before the freeze ends, '
'contact the troopers about how to proceed'),
]))]
return []
infra/config/generated/goma-usage.pyl 0 → 100644
View file @ 70ca76dc
# This is a non-LUCI generated file
# This file provides an abstract notion of how heavily goma is being used
# This is consumed by presubmit checks that need to validate the config
{
# 61150 (scheduler total) + 359500 (weighted CQ total)
# (weighted CQ total) = 7190 (CQ total) * 50 (CQ weight)
'*weighted total*': 420650,
'scheduler': {
'*total*': 61150,
'triggered': {
'*total*': 59600,
('https://chromium.googlesource.com/chromium/src', 'regexp:refs/heads/master'): {
'*total*': 58720,
# 4 concurrent builds x 80 jobs
'ci/ASAN Debug': 320,
# 5 concurrent builds x 80 jobs
'ci/ASAN Release': 400,
# 4 concurrent builds x 80 jobs
'ci/ASAN Release Media': 320,
# 4 concurrent builds x 80 jobs
'ci/ASan Debug (32-bit x86 with V8-ARM)': 320,
# 4 concurrent builds x 80 jobs
'ci/ASan Release (32-bit x86 with V8-ARM)': 320,
# 4 concurrent builds x 80 jobs
'ci/ASan Release Media (32-bit x86 with V8-ARM)': 320,
# 4 concurrent builds x 80 jobs
'ci/Afl Upload Linux ASan': 320,
'ci/Android ASAN (dbg)': 150,
'ci/Android CFI': 500,
'ci/Android FYI 32 Vk Release (Pixel 2)': 80,
'ci/Android FYI 32 dEQP Vk Release (Pixel 2)': 80,
'ci/Android FYI 64 Vk Release (Pixel 2)': 80,
'ci/Android FYI 64 dEQP Vk Release (Pixel 2)': 80,
'ci/Android FYI Release (NVIDIA Shield TV)': 80,
'ci/Android FYI Release (Nexus 5)': 80,
'ci/Android FYI Release (Nexus 5X)': 80,
'ci/Android FYI Release (Nexus 6)': 80,
'ci/Android FYI Release (Nexus 9)': 80,
'ci/Android FYI Release (Pixel 2)': 80,
'ci/Android FYI SkiaRenderer GL (Nexus 5X)': 80,
'ci/Android FYI SkiaRenderer Vulkan (Pixel 2)': 80,
'ci/Android FYI dEQP Release (Nexus 5X)': 80,
'ci/Android Release (Nexus 5X)': 80,
'ci/Android WebView P FYI (rel)': 80,
'ci/Android arm Builder (dbg)': 150,
'ci/Android arm64 Builder (dbg)': 500,
'ci/Android x64 Builder (dbg)': 150,
'ci/Android x86 Builder (dbg)': 150,
'ci/CFI Linux CF': 200,
'ci/CFI Linux ToT': 200,
'ci/Cast Android (dbg)': 150,
'ci/Cast Audio Linux': 500,
'ci/Cast Linux': 50,
'ci/ChromeOS FYI Release (amd64-generic)': 80,
'ci/ChromeOS FYI Release (kevin)': 80,
# 6 concurrent builds x 80 jobs
'ci/ChromiumOS ASAN Release': 480,
'ci/CrWinAsan': 200,
'ci/CrWinAsan(dll)': 200,
'ci/Dawn Linux x64 Builder': 80,
'ci/Dawn Linux x64 DEPS Builder': 80,
# jobs count assumes an 8-core machine
'ci/Dawn Mac x64 Builder': 80,
# jobs count assumes an 8-core machine
'ci/Dawn Mac x64 DEPS Builder': 80,
'ci/Dawn Win10 x64 ASAN Release': 80,
'ci/Dawn Win10 x64 Builder': 80,
'ci/Dawn Win10 x64 DEPS Builder': 80,
'ci/Dawn Win10 x86 Builder': 80,
'ci/Dawn Win10 x86 DEPS Builder': 80,
'ci/Deterministic Android': 150,
'ci/Deterministic Android (dbg)': 150,
'ci/Deterministic Fuchsia (dbg)': 80,
'ci/Deterministic Linux': 500,
'ci/Deterministic Linux (dbg)': 500,
'ci/Fuchsia ARM64': 500,
'ci/Fuchsia x64': 500,
'ci/GPU FYI Linux Builder': 80,
'ci/GPU FYI Linux Builder (dbg)': 80,
'ci/GPU FYI Linux Ozone Builder': 80,
'ci/GPU FYI Linux dEQP Builder': 80,
# jobs count assumes an 8-core machine
'ci/GPU FYI Mac Builder': 80,
# jobs count assumes an 8-core machine
'ci/GPU FYI Mac Builder (dbg)': 80,
# jobs count assumes an 8-core machine
'ci/GPU FYI Mac dEQP Builder': 80,
'ci/GPU FYI Perf Android 64 Builder': 80,
'ci/GPU FYI Win Builder': 80,
'ci/GPU FYI Win Builder (dbg)': 80,
'ci/GPU FYI Win dEQP Builder': 80,
'ci/GPU FYI Win x64 Builder': 80,
'ci/GPU FYI Win x64 Builder (dbg)': 80,
'ci/GPU FYI Win x64 DX12 Vulkan Builder': 80,
'ci/GPU FYI Win x64 DX12 Vulkan Builder (dbg)': 80,
'ci/GPU FYI Win x64 dEQP Builder': 80,
'ci/GPU FYI XR Win x64 Builder': 80,
'ci/GPU Linux Builder': 80,
'ci/GPU Linux Builder (dbg)': 80,
# jobs count assumes an 8-core machine
'ci/GPU Mac Builder': 80,
# jobs count assumes an 8-core machine
'ci/GPU Mac Builder (dbg)': 80,
'ci/GPU Win x64 Builder': 80,
'ci/GPU Win x64 Builder (dbg)': 80,
'ci/Leak Detection Linux': 500,
# 3 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Chrome OS ASan': 240,
# 5 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Linux ASan': 400,
# 5 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Linux ASan Debug': 400,
# 5 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Linux MSan': 400,
# 5 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Linux UBSan': 400,
'ci/Libfuzzer Upload Linux V8-ARM64 ASan': 80,
'ci/Libfuzzer Upload Linux V8-ARM64 ASan Debug': 80,
# 3 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Linux32 ASan': 240,
# 3 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Linux32 ASan Debug': 240,
'ci/Libfuzzer Upload Linux32 V8-ARM ASan': 80,
'ci/Libfuzzer Upload Linux32 V8-ARM ASan Debug': 80,
'ci/Libfuzzer Upload Mac ASan': 200,
# 3 concurrent builds x 80 jobs
'ci/Libfuzzer Upload Windows ASan': 240,
'ci/Linux ASan LSan Builder': 500,
'ci/Linux Builder': 500,
'ci/Linux Builder (dbg)': 500,
'ci/Linux Builder (dbg)(32)': 500,
'ci/Linux CFI': 500,
'ci/Linux Chromium OS ASan LSan Builder': 500,
'ci/Linux ChromiumOS Full': 80,
'ci/Linux ChromiumOS MSan Builder': 500,
'ci/Linux FYI GPU TSAN Release': 80,
'ci/Linux FYI SkiaRenderer Dawn Release (Intel HD 630)': 80,
'ci/Linux MSan Builder': 500,
'ci/Linux TSan Builder': 500,
'ci/Linux Viz': 80,
'ci/Linux remote_run Builder': 80,
# 4 concurrent builds x 80 jobs
'ci/MSAN Release (chained origins)': 320,
# 4 concurrent builds x 80 jobs
'ci/MSAN Release (no origins)': 320,
# 2 concurrent builds x 40 jobs
'ci/Mac ASAN Release': 80,
# 2 concurrent builds x 40 jobs
'ci/Mac ASAN Release Media': 80,
# 2 concurrent builds x 80 jobs
# jobs count assumes an 8-core machine
'ci/Mac ASan 64 Builder': 160,
# jobs count assumes an 8-core machine
'ci/Mac Builder': 80,
# jobs count assumes an 8-core machine
'ci/Mac Builder (dbg)': 80,
# jobs count assumes an 8-core machine
'ci/Mac Builder Next': 80,
# jobs count assumes an 8-core machine
'ci/Mac FYI GPU ASAN Release': 80,
# jobs count assumes an 8-core machine
'ci/Mac FYI arm64 Release (Apple DTK)': 80,
# jobs count assumes an 8-core machine
'ci/Mac deterministic': 80,
# jobs count assumes an 8-core machine
'ci/Mac deterministic (dbg)': 80,
'ci/Mojo Android': 80,
'ci/Mojo ChromiumOS': 80,
'ci/Mojo Linux': 80,
'ci/Mojo Windows': 80,
'ci/Network Service Linux': 500,
'ci/Site Isolation Android': 80,
# 4 concurrent builds x 80 jobs
'ci/TSAN Debug': 320,
# 3 concurrent builds x 80 jobs
'ci/TSAN Release': 240,
'ci/ToTAndroid': 200,
'ci/ToTAndroid (dbg)': 200,
'ci/ToTAndroid x64': 200,
'ci/ToTAndroid64': 200,
'ci/ToTAndroidASan': 200,
'ci/ToTAndroidCFI': 200,
'ci/ToTAndroidOfficial': 200,
'ci/ToTLinux': 200,
'ci/ToTLinux (dbg)': 200,
'ci/ToTLinuxASan': 200,
'ci/ToTLinuxASanLibfuzzer': 200,
'ci/ToTLinuxCoverage': 200,
'ci/ToTLinuxMSan': 200,
'ci/ToTLinuxTSan': 200,
'ci/ToTLinuxThinLTO': 200,
'ci/ToTLinuxUBSanVptr': 200,
'ci/ToTMac': 200,
'ci/ToTMac (dbg)': 200,
'ci/ToTMacASan': 200,
'ci/ToTMacCoverage': 200,
'ci/ToTWin(dbg)': 200,
'ci/ToTWin(dll)': 200,
'ci/ToTWin64(dbg)': 200,
'ci/ToTWin64(dll)': 200,
'ci/ToTWinASanLibfuzzer': 200,
'ci/ToTWinCFI': 200,
'ci/ToTWinCFI64': 200,
'ci/ToTWindowsCoverage': 200,
# jobs count assumes an 8-core machine
'ci/ToTiOS': 80,
# jobs count assumes an 8-core machine
'ci/ToTiOSDevice': 80,
# 4 concurrent builds x 80 jobs
'ci/UBSan Release': 320,
# 4 concurrent builds x 80 jobs
'ci/UBSan vptr Release': 320,
'ci/UBSanVptr Linux': 200,
'ci/VR Linux': 80,
'ci/WebKit Linux ASAN': 500,
'ci/WebKit Linux Leak': 500,
'ci/WebKit Linux MSAN': 500,
'ci/Win 10 Fast Ring': 80,
# 7 concurrent builds x 80 jobs
'ci/Win ASan Release': 560,
# 6 concurrent builds x 80 jobs
'ci/Win ASan Release Media': 480,
'ci/Win Builder': 200,
'ci/Win Builder (dbg)': 200,
'ci/Win x64 Builder': 200,
'ci/Win x64 Builder (dbg)': 200,
'ci/Win10 FYI x64 SkiaRenderer Dawn Release (NVIDIA)': 80,
'ci/Windows deterministic': 150,
'ci/android-10-arm64-rel': 150,
'ci/android-11-x86-fyi-rel': 80,
'ci/android-archive-dbg': 80,
'ci/android-archive-rel': 200,
'ci/android-arm64-proguard-rel': 500,
'ci/android-asan': 500,
'ci/android-bfcache-rel': 150,
'ci/android-binary-size-generator': 150,
'ci/android-code-coverage-native': 200,
'ci/android-cronet-arm-dbg': 150,
'ci/android-cronet-arm-rel': 150,
'ci/android-cronet-arm64-dbg': 150,
'ci/android-cronet-arm64-rel': 150,
'ci/android-cronet-asan-arm-rel': 150,
'ci/android-cronet-marshmallow-arm64-perf-rel': 150,
'ci/android-cronet-x86-dbg': 150,
'ci/android-cronet-x86-rel': 150,
'ci/android-incremental-dbg': 150,
'ci/android-inverse-fieldtrials-pie-x86-fyi-rel': 80,
'ci/android-lollipop-arm-rel': 150,
'ci/android-marshmallow-arm64-rel': 150,
'ci/android-marshmallow-x86-rel': 150,
'ci/android-marshmallow-x86-rel-non-cq': 150,
'ci/android-mojo-webview-rel': 80,
'ci/android-nougat-arm64-rel': 150,
'ci/android-official': 200,
'ci/android-paeverywhere-arm-fyi-dbg': 80,
'ci/android-paeverywhere-arm-fyi-rel': 80,
'ci/android-paeverywhere-arm64-fyi-dbg': 80,
'ci/android-paeverywhere-arm64-fyi-rel': 80,
'ci/android-pie-arm64-rel': 150,
'ci/android-pie-arm64-wpt-rel-non-cq': 80,
'ci/android-pie-x86-fyi-rel': 150,
'ci/android-pie-x86-rel': 150,
'ci/android-weblayer-x86-fyi-rel': 80,
'ci/chromeos-amd64-generic-asan-rel': 80,
'ci/chromeos-amd64-generic-cfi-thin-lto-rel': 80,
'ci/chromeos-amd64-generic-dbg': 80,
'ci/chromeos-amd64-generic-lacros-dbg': 80,
'ci/chromeos-amd64-generic-lacros-rel': 80,
'ci/chromeos-amd64-generic-rel': 80,
'ci/chromeos-arm-generic-dbg': 80,
'ci/chromeos-arm-generic-rel': 80,
'ci/chromeos-kevin-rel': 80,
'ci/fuchsia-arm64-cast': 500,
'ci/fuchsia-fyi-arm64-dbg': 80,
'ci/fuchsia-fyi-arm64-rel': 80,
'ci/fuchsia-fyi-x64-dbg': 80,
'ci/fuchsia-fyi-x64-rel': 80,
'ci/fuchsia-official': 200,
'ci/fuchsia-x64-cast': 500,
'ci/fuchsia-x64-dbg': 500,
# jobs count assumes an 8-core machine
'ci/ios-asan': 80,
# jobs count assumes an 8-core machine
'ci/ios-device': 80,
# jobs count assumes an 8-core machine
'ci/ios-simulator': 80,
# jobs count assumes an 8-core machine
'ci/ios-simulator-code-coverage': 80,
# jobs count assumes an 8-core machine
'ci/ios-simulator-cronet': 80,
# jobs count assumes an 8-core machine
'ci/ios-simulator-full-configs': 80,
# jobs count assumes an 8-core machine
'ci/ios-simulator-multi-window': 80,
# jobs count assumes an 8-core machine
'ci/ios-simulator-noncq': 80,
# jobs count assumes an 8-core machine
'ci/ios13-sdk-device': 80,
# jobs count assumes an 8-core machine
'ci/ios14-beta-simulator': 80,
# jobs count assumes an 8-core machine
'ci/ios14-sdk-simulator': 80,
'ci/linux-annotator-rel': 80,
'ci/linux-archive-dbg': 80,
'ci/linux-archive-rel': 200,
'ci/linux-ash-chromium-builder-fyi-rel': 80,
'ci/linux-bfcache-rel': 500,
'ci/linux-blink-animation-use-time-delta': 80,
'ci/linux-blink-heap-concurrent-marking-tsan-rel': 80,
'ci/linux-blink-heap-verification': 80,
'ci/linux-chromeos-dbg': 80,
'ci/linux-chromeos-rel': 80,
'ci/linux-chromium-tests-staging-builder': 80,
'ci/linux-fieldtrial-rel': 80,
'ci/linux-gcc-rel': 500,
'ci/linux-inverse-fieldtrials-fyi-rel': 80,
'ci/linux-lacros-builder-fyi-rel': 80,
'ci/linux-lacros-builder-rel': 80,
'ci/linux-official': 200,
'ci/linux-ozone-rel': 500,
'ci/linux-perfetto-rel': 80,
'ci/linux-swangle-chromium-x64': 80,
'ci/linux-swangle-tot-angle-x64': 80,
'ci/linux-swangle-tot-angle-x86': 80,
'ci/linux-swangle-tot-swiftshader-x64': 80,
'ci/linux-swangle-tot-swiftshader-x86': 80,
'ci/linux-swangle-x64': 80,
'ci/linux-swangle-x86': 80,
'ci/linux-trusty-rel': 500,
'ci/linux-upload-perfetto': 80,
'ci/linux-win_cross-rel': 200,
'ci/linux-wpt-fyi-rel': 80,
'ci/linux-wpt-identity-fyi-rel': 80,
'ci/linux-wpt-input-fyi-rel': 80,
'ci/linux-wpt-payments-fyi-rel': 80,
'ci/mac-archive-dbg': 40,
'ci/mac-archive-rel': 80,
# jobs count assumes an 8-core machine
'ci/mac-arm64-rel': 80,
'ci/mac-code-coverage': 200,
'ci/mac-hermetic-upgrade-rel': 80,
'ci/mac-mojo-rel': 40,
# jobs count assumes an 8-core machine
'ci/mac-official': 80,
# jobs count assumes an 8-core machine
'ci/mac-swangle-chromium-x64': 80,
# jobs count assumes an 8-core machine
'ci/mac-updater-builder-dbg': 80,
# jobs count assumes an 8-core machine
'ci/mac-updater-builder-rel': 80,
'ci/metadata-exporter': 500,
'ci/win-annotator-rel': 80,
'ci/win-archive-dbg': 200,
'ci/win-archive-rel': 200,
'ci/win-asan': 500,
'ci/win-official': 200,
'ci/win-paeverywhere-x64-fyi-dbg': 80,
'ci/win-paeverywhere-x64-fyi-rel': 80,
'ci/win-paeverywhere-x86-fyi-dbg': 80,
'ci/win-paeverywhere-x86-fyi-rel': 80,
'ci/win-pixel-builder-rel': 80,
'ci/win-swangle-chromium-x86': 80,
'ci/win-swangle-tot-angle-x64': 80,
'ci/win-swangle-tot-angle-x86': 80,
'ci/win-swangle-tot-swiftshader-x64': 80,
'ci/win-swangle-tot-swiftshader-x86': 80,
'ci/win-swangle-x64': 80,
'ci/win-swangle-x86': 80,
'ci/win-updater-builder-dbg': 80,
'ci/win-updater-builder-rel': 80,
'ci/win10-code-coverage': 200,
'ci/win10-inverse-fieldtrials-fyi-rel': 80,
'ci/win32-archive-dbg': 200,
'ci/win32-archive-rel': 200,
'ci/win32-arm64-rel': 150,
'ci/win32-official': 200,
'ci/win32-updater-builder-dbg': 80,
'ci/win32-updater-builder-rel': 80,
'goma/Chromium Android ARM 32-bit Goma RBE Staging': 80,
'goma/Chromium Android ARM 32-bit Goma RBE ToT': 80,
'goma/Chromium Android ARM 32-bit Goma RBE ToT (ATS)': 80,
'goma/Chromium Linux Goma RBE Staging': 80,
'goma/Chromium Linux Goma RBE Staging (clobber)': 80,
'goma/Chromium Linux Goma RBE Staging (dbg)': 80,
'goma/Chromium Linux Goma RBE Staging (dbg) (clobber)': 80,
'goma/Chromium Linux Goma RBE ToT': 80,
'goma/Chromium Linux Goma RBE ToT (ATS)': 80,
'goma/Chromium Linux Goma Staging': 80,
'goma/Chromium Mac Goma RBE Staging': 80,
'goma/Chromium Mac Goma RBE Staging (clobber)': 80,
'goma/Chromium Mac Goma RBE Staging (dbg)': 80,
'goma/Chromium Mac Goma RBE ToT': 80,
'goma/Chromium Mac Goma Staging': 80,
'goma/Chromium Win Goma RBE Staging': 200,
'goma/Chromium Win Goma RBE Staging (clobber)': 200,
'goma/Chromium Win Goma RBE ToT': 200,
'goma/Chromium iOS Goma RBE ToT': 80,
'goma/CrWinGomaStaging': 80,
'goma/Linux Builder Goma Canary': 80,
'goma/Linux Builder Goma Latest Client': 80,
'goma/Linux Builder Goma RBE Canary': 80,
'goma/Linux Builder Goma RBE Latest Client': 80,
'goma/Mac Builder (dbg) Goma Canary': 40,
'goma/Mac Builder (dbg) Goma Canary (clobber)': 40,
'goma/Mac Builder (dbg) Goma Latest Client': 40,
'goma/Mac Builder (dbg) Goma Latest Client (clobber)': 40,
'goma/Mac Builder (dbg) Goma RBE Canary (clobber)': 80,
'goma/Mac Builder (dbg) Goma RBE Latest Client (clobber)': 80,
'goma/Mac Builder Goma Canary': 40,
'goma/Mac Builder Goma Latest Client': 40,
'goma/Win Builder (dbg) Goma Canary': 80,
'goma/Win Builder (dbg) Goma Latest Client': 80,
'goma/Win Builder (dbg) Goma RBE Latest Client': 80,
'goma/Win Builder Goma Canary': 80,
'goma/Win Builder Goma Latest Client': 80,
'goma/Win Builder Goma RBE Latest Client': 80,
'goma/Win7 Builder (dbg) Goma Canary': 80,
'goma/Win7 Builder (dbg) Goma Latest Client': 80,
'goma/Win7 Builder Goma Canary': 80,
'goma/Win7 Builder Goma Latest Client': 80,
'goma/android-archive-dbg-goma-canary': 80,
'goma/android-archive-dbg-goma-latest': 80,
'goma/android-archive-dbg-goma-rbe-ats-canary': 80,
'goma/android-archive-dbg-goma-rbe-ats-latest': 80,
'goma/android-archive-dbg-goma-rbe-canary': 80,
'goma/android-archive-dbg-goma-rbe-latest': 80,
'goma/chromeos-amd64-generic-rel-goma-canary': 80,
'goma/chromeos-amd64-generic-rel-goma-latest': 80,
'goma/chromeos-amd64-generic-rel-goma-rbe-canary': 80,
'goma/chromeos-amd64-generic-rel-goma-rbe-latest': 80,
'goma/chromeos-amd64-generic-rel-goma-rbe-staging': 80,
'goma/chromeos-amd64-generic-rel-goma-rbe-tot': 80,
# jobs count assumes an 8-core machine
'goma/ios-device-goma-canary-clobber': 80,
# jobs count assumes an 8-core machine
'goma/ios-device-goma-latest-clobber': 80,
# jobs count assumes an 8-core machine
'goma/ios-device-goma-rbe-canary-clobber': 80,
# jobs count assumes an 8-core machine
'goma/ios-device-goma-rbe-latest-clobber': 80,
'goma/linux-archive-rel-goma-canary': 80,
'goma/linux-archive-rel-goma-canary-localoutputcache': 80,
'goma/linux-archive-rel-goma-latest': 80,
'goma/linux-archive-rel-goma-latest-localoutputcache': 80,
'goma/linux-archive-rel-goma-rbe-ats-canary': 80,
'goma/linux-archive-rel-goma-rbe-ats-latest': 80,
'goma/linux-archive-rel-goma-rbe-canary': 80,
'goma/linux-archive-rel-goma-rbe-latest': 80,
'goma/mac-archive-rel-goma-canary': 40,
'goma/mac-archive-rel-goma-canary-localoutputcache': 40,
'goma/mac-archive-rel-goma-latest': 40,
'goma/mac-archive-rel-goma-latest-localoutputcache': 40,
'goma/mac-archive-rel-goma-rbe-canary': 80,
'goma/mac-archive-rel-goma-rbe-latest': 80,
'goma/win32-archive-rel-goma-canary-localoutputcache': 80,
'goma/win32-archive-rel-goma-latest-localoutputcache': 80,
# jobs count assumes an 8-core machine
'webrtc/WebRTC Chromium Android Builder': 80,
# jobs count assumes an 8-core machine
'webrtc/WebRTC Chromium Linux Builder': 80,
'webrtc/WebRTC Chromium Mac Builder': 80,
# jobs count assumes an 8-core machine
'webrtc/WebRTC Chromium Win Builder': 80,
},
('https://webrtc.googlesource.com/src/', 'regexp:refs/heads/master'): {
'*total*': 880,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI Android Builder': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI Android Builder (dbg)': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI Android Builder ARM64 (dbg)': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI Linux Builder': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI Linux Builder (dbg)': 80,
'webrtc.fyi/WebRTC Chromium FYI Mac Builder': 80,
'webrtc.fyi/WebRTC Chromium FYI Mac Builder (dbg)': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI Win Builder': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI Win Builder (dbg)': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI ios-device': 80,
# jobs count assumes an 8-core machine
'webrtc.fyi/WebRTC Chromium FYI ios-simulator': 80,
},
},
'scheduled': {
'*total*': 1550,
'0 0,12 * * *': {
'*total*': 80,
# jobs count assumes an 8-core machine
'ci/ios13-beta-simulator': 80,
},
'0 0,6,12,18 * * *': {
'*total*': 160,
'ci/win-celab-builder-rel': 80,
# jobs count assumes an 8-core machine
'findit/linux_chromium_bot_db_exporter': 80,
},
'0 1-23/6 * * *': {
'*total*': 80,
# jobs count assumes an 8-core machine
'ci/ios-webkit-tot': 80,
},
'0 6,18 * * *': {
'*total*': 80,
# jobs count assumes an 8-core machine
'ci/ios13-sdk-simulator': 80,
},
'0 7 * * 0 *': {
'*total*': 160,
'ci/android-avd-packager': 80,
'ci/android-sdk-packager': 80,
},
'0 7,14,22 * * * *': {
'*total*': 80,
'ci/android-androidx-packager': 80,
},
'triggered': {
'*total*': 750,
'ci/android-code-coverage': 200,
'ci/android-pie-x86-fyi-rel': 150,
'ci/linux-chromeos-code-coverage': 200,
'ci/linux-chromeos-js-code-coverage': 200,
},
'with 3h interval': {
'*total*': 160,
'ci/mac-upload-perfetto': 80,
'ci/win-upload-perfetto': 80,
},
},
},
'cq': {
'*total*': 7190,
'try/android-binary-size': 150,
'try/android-cronet-arm-dbg': 80,
'try/android-lollipop-arm-rel': 150,
'try/android-marshmallow-arm64-rel': 300,
'try/android-marshmallow-x86-rel': 300,
'try/android-pie-arm64-dbg': 300,
'try/android-pie-arm64-rel': 300,
'try/android_compile_dbg': 150,
'try/android_compile_x64_dbg': 80,
'try/android_compile_x86_dbg': 80,
'try/android_cronet': 80,
'try/android_optional_gpu_tests_rel': 80,
'try/cast_shell_android': 80,
'try/cast_shell_linux': 80,
'try/chromeos-amd64-generic-dbg': 80,
'try/chromeos-amd64-generic-rel': 80,
'try/chromeos-arm-generic-rel': 80,
'try/chromeos-kevin-rel': 80,
'try/chromium_presubmit': 80,
# jobs count assumes an 8-core machine
'try/dawn-linux-x64-deps-rel': 80,
# jobs count assumes an 8-core machine
'try/dawn-mac-x64-deps-rel': 80,
# jobs count assumes an 8-core machine
'try/dawn-win10-x64-deps-rel': 80,
# jobs count assumes an 8-core machine
'try/dawn-win10-x86-deps-rel': 80,
'try/fuchsia-arm64-cast': 80,
'try/fuchsia-compile-x64-dbg': 80,
'try/fuchsia-x64-cast': 80,
'try/fuchsia_arm64': 80,
'try/fuchsia_x64': 80,
# jobs count assumes an 8-core machine
'try/ios-simulator': 80,
# jobs count assumes an 8-core machine
'try/ios-simulator-coverage-exp': 80,
# jobs count assumes an 8-core machine
'try/ios-simulator-cronet': 80,
# jobs count assumes an 8-core machine
'try/ios-simulator-full-configs': 80,
# jobs count assumes an 8-core machine
'try/ios-simulator-full-configs-coverage-exp': 80,
'try/linux-blink-rel': 80,
'try/linux-chromeos-compile-dbg': 80,
'try/linux-chromeos-rel': 150,
'try/linux-lacros-rel': 300,
'try/linux-libfuzzer-asan-rel': 80,
'try/linux-ozone-rel': 80,
'try/linux-perfetto-rel': 80,
'try/linux-rel': 150,
'try/linux-rel-builderful': 150,
'try/linux_chromium_asan_rel_ng': 150,
'try/linux_chromium_compile_dbg_ng': 150,
'try/linux_chromium_dbg_ng': 80,
'try/linux_chromium_tsan_rel_ng': 150,
'try/linux_layout_tests_composite_after_paint': 80,
'try/linux_layout_tests_layout_ng_disabled': 80,
'try/linux_optional_gpu_tests_rel': 80,
'try/linux_vr': 80,
'try/mac-rel': 150,
# jobs count assumes an 8-core machine
'try/mac-updater-try-builder-dbg': 80,
# jobs count assumes an 8-core machine
'try/mac-updater-try-builder-rel': 80,
'try/mac_chromium_compile_dbg_ng': 150,
# jobs count assumes an 8-core machine
'try/mac_optional_gpu_tests_rel': 80,
'try/win-libfuzzer-asan-rel': 80,
'try/win-updater-try-builder-dbg': 80,
'try/win-updater-try-builder-rel': 80,
'try/win10_chromium_x64_rel_ng': 150,
'try/win7-rel': 300,
'try/win_chromium_compile_dbg_ng': 150,
'try/win_optional_gpu_tests_rel': 80,
},
}
infra/config/generators/goma-usage.star 0 → 100644
View file @ 70ca76dc
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
load("//project.star", "settings")
def _get_builder_id(bucket, builder):
return "{}/{}".format(bucket, builder)
def _get_goma_usage(builder):
properties = json.decode(builder.properties)
jobs = properties.get("$build/goma", {}).get("jobs")
if jobs != None:
return struct(jobs = jobs, estimate = False)
# If the dimensions don't include the cores, guess 8
cores = 8
estimate = True
for d in builder.dimensions:
if d.startswith("cores:"):
cores = int(d[len("cores:"):])
estimate = False
break
# default logic taken from
# https://source.chromium.org/chromium/chromium/tools/build/+/master:recipes/recipe_modules/goma/api.py;l=158;drc=d29660fdde91466d1a061e3fe85ea77b51951f00
jobs = cores * 10
if jobs > 200:
jobs = 200
return struct(jobs = jobs, estimate = estimate)
def _get_builder_goma_usage(ctx):
goma_usage_by_builder = {}
for bucket in ctx.output["cr-buildbucket.cfg"].buckets:
for builder in bucket.swarming.builders:
builder_id = _get_builder_id(bucket.name, builder.name)
goma_usage_by_builder[builder_id] = _get_goma_usage(builder)
return goma_usage_by_builder
def _get_scheduler_goma_usage(ctx, goma_usage_by_builder):
scheduler_cfg = ctx.output["luci-scheduler.cfg"]
gitiles_by_job = {}
for trigger in scheduler_cfg.trigger:
gitiles = {
(trigger.gitiles.repo, ref): True
for ref in trigger.gitiles.refs
}
for job in trigger.triggers:
gitiles_by_job.setdefault(job, {}).update(gitiles)
total_gitiles_goma_usage = 0
total_goma_usage_by_gitiles = {}
goma_usage_by_builder_by_gitiles = {}
total_schedule_goma_usage = 0
total_goma_usage_by_schedule = {}
goma_usage_by_builder_by_schedule = {}
for job in scheduler_cfg.job:
if job.id not in gitiles_by_job and not job.schedule:
continue
if not job.buildbucket.bucket:
continue
_, project, bucket = job.buildbucket.bucket.split(".", 2)
if project != settings.project:
continue
builder_id = _get_builder_id(bucket, job.buildbucket.builder)
goma_usage = goma_usage_by_builder[builder_id]
concurrent = 1
if (proto.has(job, "triggering_policy") and
proto.has(job.triggering_policy, "max_concurrent_invocations")):
concurrent = job.triggering_policy.max_concurrent_invocations
goma_jobs = goma_usage.jobs * concurrent
goma_usage = struct(
jobs = goma_jobs,
jobs_per_build = goma_usage.jobs,
estimate = goma_usage.estimate,
concurrent = concurrent,
)
for g in gitiles_by_job.get(job.id, []):
gitiles_dict = goma_usage_by_builder_by_gitiles.setdefault(g, {})
gitiles_dict[builder_id] = goma_usage
total_goma_usage_by_gitiles.setdefault(g, 0)
total_goma_usage_by_gitiles[g] += goma_jobs
total_gitiles_goma_usage += goma_jobs
if job.schedule:
schedule_dict = goma_usage_by_builder_by_schedule.setdefault(job.schedule, {})
schedule_dict[builder_id] = goma_usage
total_goma_usage_by_schedule.setdefault(job.schedule, 0)
total_goma_usage_by_schedule[job.schedule] += goma_jobs
total_schedule_goma_usage += goma_jobs
gitiles_goma_usage = struct(
total = total_gitiles_goma_usage,
by_gitiles = {
k: struct(
total = total_goma_usage_by_gitiles[k],
by_builder = goma_usage_by_builder_by_gitiles[k],
)
for k in total_goma_usage_by_gitiles
},
)
schedule_goma_usage = struct(
total = total_schedule_goma_usage,
by_schedule = {
k: struct(
total = total_goma_usage_by_schedule[k],
by_builder = goma_usage_by_builder_by_schedule[k],
)
for k in total_goma_usage_by_schedule
},
)
return gitiles_goma_usage, schedule_goma_usage
def _get_cq_goma_usage(ctx, goma_usage_by_builder):
cq_cfg = ctx.output["commit-queue.cfg"]
total_cq_goma_usage = 0
cq_goma_usage_by_builder = {}
for group in cq_cfg.config_groups:
if not proto.has(group.verifiers, "tryjob"):
continue
for builder in group.verifiers.tryjob.builders:
if builder.includable_only:
continue
project, builder_id = builder.name.split("/", 1)
if project != settings.project:
continue
goma_usage = goma_usage_by_builder[builder_id]
total_cq_goma_usage += goma_usage.jobs
cq_goma_usage_by_builder[builder_id] = goma_usage
return struct(
total = total_cq_goma_usage,
by_builder = cq_goma_usage_by_builder,
)
def _pyl_formatter(output):
indent = [""]
def add_line(*lines):
for line in lines:
line = line.strip()
if line:
if line[0] in ")}]":
indent[0] = indent[0][:-2]
output.append(indent[0])
output.append(line)
if line[-1] in "[{(":
indent[0] += " "
output.append("\n")
return add_line
# About 150 concurrent CQ attempts, assume 1/3 is spent compiling
_CQ_WEIGHT = 50
def _generate_goma_usage(ctx):
goma_usage_by_builder = _get_builder_goma_usage(ctx)
gitiles_goma_usage, schedule_goma_usage = (
_get_scheduler_goma_usage(ctx, goma_usage_by_builder)
)
cq_goma_usage = _get_cq_goma_usage(ctx, goma_usage_by_builder)
output = []
_ = _pyl_formatter(output)
_(
"# This is a non-LUCI generated file",
"# This file provides an abstract notion of how heavily goma is being used",
"# This is consumed by presubmit checks that need to validate the config",
"",
)
scheduler_total = gitiles_goma_usage.total + schedule_goma_usage.total
weighted_cq_total = cq_goma_usage.total * _CQ_WEIGHT
_(
"{",
"# {scheduler_total} (scheduler total) + {weighted_cq_total} (weighted CQ total)".format(
scheduler_total = scheduler_total,
weighted_cq_total = weighted_cq_total,
),
"# (weighted CQ total) = {cq_total} (CQ total) * {cq_weight} (CQ weight)".format(
cq_total = cq_goma_usage.total,
cq_weight = _CQ_WEIGHT,
),
"'*weighted total*': {},".format(scheduler_total + cq_goma_usage.total * _CQ_WEIGHT),
)
_(
"'scheduler': {",
"'*total*': {},".format(scheduler_total),
)
def _output_scheduler_goma_usage(name, goma_usage):
_(
"{}: {{".format(name),
"'*total*': {},".format(goma_usage.total),
)
for builder, goma_usage_for_builder in sorted(goma_usage.by_builder.items()):
if goma_usage_for_builder.concurrent > 1:
_("# {concurrent} concurrent builds x {jobs} jobs".format(
concurrent = goma_usage_for_builder.concurrent,
jobs = goma_usage_for_builder.jobs_per_build,
))
if goma_usage_for_builder.estimate:
_("# jobs count assumes an 8-core machine")
_("'{}': {},".format(builder, goma_usage_for_builder.jobs))
# Close the dict for name
_("},")
_(
"'triggered': {",
"'*total*': {},".format(gitiles_goma_usage.total),
)
for gitiles, goma_usage_for_gitiles in sorted(gitiles_goma_usage.by_gitiles.items()):
name = "('{}', '{}')".format(*gitiles)
_output_scheduler_goma_usage(name, goma_usage_for_gitiles)
# Close the 'triggered' dict
_("},")
_(
"'scheduled': {",
"'*total*': {},".format(schedule_goma_usage.total),
)
for schedule, goma_usage_for_schedule in sorted(schedule_goma_usage.by_schedule.items()):
name = "'{}'".format(schedule)
_output_scheduler_goma_usage(name, goma_usage_for_schedule)
# Close the 'scheduled' dict
_("},")
# Close the 'scheduler' dict
_("},")
_(
"'cq': {",
"'*total*': {},".format(cq_goma_usage.total),
)
for builder, goma_usage in sorted(cq_goma_usage.by_builder.items()):
if goma_usage.estimate:
_("# jobs count assumes an 8-core machine")
_("'{}': {},".format(builder, goma_usage.jobs))
# Close the 'cq' dict
_("},")
_(
# Close the top-level dict
"}",
# End with a newline
"",
)
ctx.output["goma-usage.pyl"] = "".join(output)
lucicfg.generator(_generate_goma_usage)
infra/config/main.star
View file @ 70ca76dc
......@@ -10,7 +10,7 @@ load("//lib/branches.star", "branches")
load("//project.star", "settings")
lucicfg.check_version(
min = "1.19.0",
min = "1.21.0",
message = "Update depot_tools",
)
......@@ -27,6 +27,7 @@ lucicfg.config(
"commit-queue.cfg",
"cq-builders.md",
"cr-buildbucket.cfg",
"goma-usage.pyl",
"luci-logdog.cfg",
"luci-milo.cfg",
"luci-notify.cfg",
......@@ -134,7 +135,7 @@ branches.exec("//subprojects/goma/subproject.star")
branches.exec("//subprojects/webrtc/subproject.star")
branches.exec("//generators/cq-builders-md.star")
exec("//generators/goma-usage.star")
exec("//generators/scheduler-noop-jobs.star")
exec("//generators/sort-consoles.star")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment